From aa9348563c88b984d9c42c41aa5d227145c8ac4b Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Mon, 3 Feb 2014 22:22:07 +0100
Subject: [PATCH 1/4] gitian: Make linux build of OpenSSL deterministic

OpenSSL was embedding a timestamp causing its build to be
non-deterministic.
Change deps-linux to be deterministic by using FAKETIME
as needed and disabling it when it gets in the way.
---
 contrib/gitian-descriptors/deps-linux.yml   | 22 +++++++++++++++++----
 contrib/gitian-descriptors/gitian-linux.yml |  6 +++---
 2 files changed, 21 insertions(+), 7 deletions(-)

diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index 12c3179ea..fc624d899 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -24,6 +24,9 @@ files:
 script: |
   STAGING="$HOME/install"
   OPTFLAGS='-O2'
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
+  export FAKETIME=$REFERENCE_DATETIME
+  export TZ=UTC
   export LIBRARY_PATH="$STAGING/lib"
   # Integrity Check
   echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3  openssl-1.0.1e.tar.gz"  | sha256sum -c
@@ -37,6 +40,7 @@ script: |
   cd openssl-1.0.1e
   #   need -fPIC to avoid relocation error in 64 bit builds
   ./config no-shared no-zlib no-dso no-krb5 --openssldir=$STAGING -fPIC
+  #   need to build OpenSSL with faketime because a timestamp is embedded into cversion.o
   make
   make install_sw
   cd ..
@@ -48,18 +52,26 @@ script: |
   rm -f $STAGING/lib/libminiupnpc.so* # no way to skip shared lib build
   cd ..
   #
-  tar xjfm qrencode-3.4.3.tar.bz2
+  tar xjf qrencode-3.4.3.tar.bz2
   cd qrencode-3.4.3
+  unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-maintainer-mode --disable-dependency-tracking
+  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking --without-zlib
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
+  export FAKETIME=$REFERENCE_DATETIME
   make $MAKEOPTS install
   cd ..
   #
-  tar xjfm protobuf-2.5.0.tar.bz2
+  tar xjf protobuf-2.5.0.tar.bz2
   cd protobuf-2.5.0
   mkdir -p $STAGING/host/bin
+  unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
   ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
+  export FAKETIME=$REFERENCE_DATETIME
   make $MAKEOPTS install
   cd ..
   #
@@ -67,9 +79,11 @@ script: |
   cd db-4.8.30.NC/build_unix
   #   need --with-pic to avoid relocation error in 64 bit builds
   ../dist/configure --prefix=$STAGING --enable-cxx --disable-shared --with-pic
+  # Workaround to prevent re-configuring by make; make all files have a date in the past
+  find . -print0 | xargs -r0 touch -t 200001010000
   make $MAKEOPTS library_build
   make install_lib install_include
   cd ../..
   #
   cd $STAGING
-  zip -r $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip include lib bin host
+  find include lib bin host -type f | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 329f4c6b4..763619280 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -21,8 +21,8 @@ remotes:
 - "url": "https://github.com/bitcoin/bitcoin.git"
   "dir": "bitcoin"
 files:
-- "bitcoin-deps-linux32-gitian-r2.zip"
-- "bitcoin-deps-linux64-gitian-r2.zip"
+- "bitcoin-deps-linux32-gitian-r3.zip"
+- "bitcoin-deps-linux64-gitian-r3.zip"
 - "boost-linux32-1.55.0-gitian-r1.zip"
 - "boost-linux64-1.55.0-gitian-r1.zip"
 script: |
@@ -34,7 +34,7 @@ script: |
   #
   mkdir -p $STAGING
   cd $STAGING
-  unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r2.zip
+  unzip ../build/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
   unzip ../build/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
   cd ../build
   #

From aabcd11ba69022c6ae24c36d56e107b530fee8ad Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Tue, 4 Feb 2014 11:55:19 +0100
Subject: [PATCH 2/4] gitian: Make linux boost dependency completely
 deterministic

It appears that the output was different every time.

This doesn't affect the final bitcoind/bitcoin-qt, but is confusing
nevertheless.

Fix it by using FAKETIME and zipping files in deterministic order.
---
 contrib/gitian-descriptors/boost-linux.yml | 9 +++++----
 contrib/gitian-descriptors/deps-linux.yml  | 4 ++--
 2 files changed, 7 insertions(+), 6 deletions(-)

diff --git a/contrib/gitian-descriptors/boost-linux.yml b/contrib/gitian-descriptors/boost-linux.yml
index 48b27aa26..d752a906a 100644
--- a/contrib/gitian-descriptors/boost-linux.yml
+++ b/contrib/gitian-descriptors/boost-linux.yml
@@ -20,11 +20,14 @@ files:
 script: |
   STAGING="$HOME/install"
   export LIBRARY_PATH="$STAGING/lib"
+  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
+  export FAKETIME=$REFERENCE_DATETIME
+  export TZ=UTC
   # Input Integrity Check
   echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52  boost_1_55_0.tar.bz2" | shasum -c
 
   mkdir -p "$STAGING"
-  tar xjf boost_1_55_0.tar.bz2
+  tar --warning=no-timestamp -xjf boost_1_55_0.tar.bz2
   cd boost_1_55_0
   GCCVERSION=$(g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2)
   # note: bjam with -d+2 reveals that -O3 is implied by default, no need to provide it in cxxflags
@@ -38,6 +41,4 @@ script: |
   ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS install
 
   cd "$STAGING"
-  export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
-  export FAKETIME=$REFERENCE_DATETIME
-  zip -r $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip *
+  find -type f | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index fc624d899..858566145 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -56,7 +56,7 @@ script: |
   cd qrencode-3.4.3
   unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking --without-zlib
+  ./configure --prefix=$STAGING --enable-static --disable-shared --with-pic --without-tools --disable-dependency-tracking
   # Workaround to prevent re-configuring by make; make all files have a date in the past
   find . -print0 | xargs -r0 touch -t 200001010000
   export FAKETIME=$REFERENCE_DATETIME
@@ -68,7 +68,7 @@ script: |
   mkdir -p $STAGING/host/bin
   unset FAKETIME # unset fake time during configure, as it does some clock sanity tests
   #   need --with-pic to avoid relocation error in 64 bit builds
-  ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic
+  ./configure --prefix=$STAGING --bindir=$STAGING/host/bin --enable-static --disable-shared --with-pic --without-zlib
   # Workaround to prevent re-configuring by make; make all files have a date in the past
   find . -print0 | xargs -r0 touch -t 200001010000
   export FAKETIME=$REFERENCE_DATETIME

From 1552145ae56bf12f3deaef1cdd8a78b2e923a80e Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Wed, 5 Feb 2014 16:44:58 +0100
Subject: [PATCH 3/4] gitian: Sort generated source distribution archive

Sort the filenames in the resulting tar, normalize the time/date
and user/group information.
---
 contrib/gitian-descriptors/gitian-linux.yml | 13 ++++++++++---
 1 file changed, 10 insertions(+), 3 deletions(-)

diff --git a/contrib/gitian-descriptors/gitian-linux.yml b/contrib/gitian-descriptors/gitian-linux.yml
index 763619280..bf355fd82 100644
--- a/contrib/gitian-descriptors/gitian-linux.yml
+++ b/contrib/gitian-descriptors/gitian-linux.yml
@@ -29,6 +29,8 @@ script: |
   STAGING="$HOME/install"
   OPTFLAGS='-O2'
   BINDIR="${OUTDIR}/bin/${GBUILD_BITS}" # 32/64 bit build specific output directory
+  TEMPDIR="$HOME/tempdir"
+  export TZ=UTC
   export LIBRARY_PATH="$STAGING/lib"
   mkdir -p ${BINDIR}
   #
@@ -39,15 +41,20 @@ script: |
   cd ../build
   #
   cd bitcoin
-  export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'`
   ./autogen.sh
   ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt"
   make dist
+  DISTNAME=`echo bitcoin-*.tar.gz`
   mkdir -p distsrc
   cd distsrc
-  tar --strip-components=1 -xf ../bitcoin-*.tar.*
+  tar --strip-components=1 -xf ../$DISTNAME
   ./configure --prefix=$STAGING --bindir=$BINDIR --with-protoc-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --disable-dependency-tracking PKG_CONFIG_PATH="$STAGING/lib/pkgconfig" CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}" BOOST_CHRONO_EXTRALIBS="-lrt"
   make $MAKEOPTS
   make $MAKEOPTS install-strip
+  
+  # sort distribution tar file and normalize user/group/mtime information for deterministic output
   mkdir -p $OUTDIR/src
-  cp ../bitcoin-*.tar.* $OUTDIR/src
+  rm -rf $TEMPDIR
+  mkdir -p $TEMPDIR
+  cd $TEMPDIR
+  tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME

From c13a13efec0051e1fe84749c36c6676dc6e267af Mon Sep 17 00:00:00 2001
From: "Wladimir J. van der Laan" <laanwj@gmail.com>
Date: Thu, 6 Feb 2014 11:57:45 +0100
Subject: [PATCH 4/4] gitian: add -D flag to ar for deterministic output for
 linux deps

ar -D: Operate in deterministic mode. When adding files and the archive
index use zero for UIDs, GIDs, timestamps, and use consistent file modes
for all files.  When this option is used, if ar is used with identical
options and identical input files, multiple runs will create identical
output files regardless of the input files' owners, groups, file modes,
or modification times.
---
 contrib/gitian-descriptors/boost-linux.yml | 14 ++++++++++++--
 contrib/gitian-descriptors/deps-linux.yml  | 11 ++++++++++-
 2 files changed, 22 insertions(+), 3 deletions(-)

diff --git a/contrib/gitian-descriptors/boost-linux.yml b/contrib/gitian-descriptors/boost-linux.yml
index d752a906a..a538ff30a 100644
--- a/contrib/gitian-descriptors/boost-linux.yml
+++ b/contrib/gitian-descriptors/boost-linux.yml
@@ -19,6 +19,7 @@ files:
 - "boost_1_55_0.tar.bz2"
 script: |
   STAGING="$HOME/install"
+  TEMPDIR="$HOME/tmp"
   export LIBRARY_PATH="$STAGING/lib"
   export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
@@ -38,7 +39,16 @@ script: |
 
   ./bootstrap.sh --without-icu
 
-  ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS install
+  ./bjam toolset=gcc threadapi=pthread threading=multi variant=release link=static runtime-link=shared --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 --layout=tagged --build-type=complete --prefix="$STAGING" $MAKEOPTS -d+2 install
 
+  # post-process all generated libraries to be deterministic
+  # extract them to a temporary directory then re-build them deterministically
+  for LIB in $(find $STAGING -name \*.a); do
+    rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+    ar xv $LIB | cut -b5- > /tmp/list.txt
+    rm $LIB
+    ar crsD $LIB $(cat /tmp/list.txt)
+  done
+  #
   cd "$STAGING"
-  find -type f | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
+  find | sort | zip -X@ $OUTDIR/boost-linux${GBUILD_BITS}-1.55.0-gitian-r1.zip
diff --git a/contrib/gitian-descriptors/deps-linux.yml b/contrib/gitian-descriptors/deps-linux.yml
index 858566145..908f9321a 100644
--- a/contrib/gitian-descriptors/deps-linux.yml
+++ b/contrib/gitian-descriptors/deps-linux.yml
@@ -23,6 +23,7 @@ files:
 - "db-4.8.30.NC.tar.gz"
 script: |
   STAGING="$HOME/install"
+  TEMPDIR="$HOME/tmp"
   OPTFLAGS='-O2'
   export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
   export FAKETIME=$REFERENCE_DATETIME
@@ -84,6 +85,14 @@ script: |
   make $MAKEOPTS library_build
   make install_lib install_include
   cd ../..
+  # post-process all generated libraries to be deterministic
+  # extract them to a temporary directory then re-build them deterministically
+  for LIB in $(find $STAGING -name \*.a); do
+    rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
+    ar xv $LIB | cut -b5- > /tmp/list.txt
+    rm $LIB
+    ar crsD $LIB $(cat /tmp/list.txt)
+  done
   #
   cd $STAGING
-  find include lib bin host -type f | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip
+  find include lib bin host | sort | zip -X@ $OUTDIR/bitcoin-deps-linux${GBUILD_BITS}-gitian-r3.zip