Browse Source

Merge pull request #3625

4ce9106 gitian: sort generated source distribution archive for windows (Wladimir J. van der Laan)
6b55e6b gitian: Post-process .a libraries for win to be deterministic (Wladimir J. van der Laan)
c43d1e5 gitian: Make windows dependencies outputs fully deterministic (Wladimir J. van der Laan)
0.10
Wladimir J. van der Laan 11 years ago
parent
commit
3e3c25dd63
No known key found for this signature in database
GPG Key ID: 74810B012346C9A6
  1. 19
      contrib/gitian-descriptors/boost-win.yml
  2. 15
      contrib/gitian-descriptors/deps-win.yml
  3. 19
      contrib/gitian-descriptors/gitian-win.yml
  4. 15
      contrib/gitian-descriptors/protobuf-win.yml
  5. 33
      contrib/gitian-descriptors/qt-win.yml

19
contrib/gitian-descriptors/boost-win.yml

@ -16,8 +16,11 @@ files: @@ -16,8 +16,11 @@ files:
- "boost-mingw-gas-cross-compile-2013-03-03.patch"
script: |
# Defines
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
export TZ=UTC
INDIR=$HOME/build
TEMPDIR=$HOME/tmp
# Input Integrity Check
echo "fff00023dd79486d444c8e29922f4072e1d451fc5a4d2b6075852ead7f2b7b52 boost_1_55_0.tar.bz2" | shasum -c
echo "d2b7f6a1d7051faef3c9cf41a92fa3671d905ef1e1da920d07651a43299f6268 boost-mingw-gas-cross-compile-2013-03-03.patch" | shasum -c
@ -35,7 +38,7 @@ script: | @@ -35,7 +38,7 @@ script: |
mkdir -p $INSTALLPREFIX $BUILDDIR
cd $BUILDDIR
#
tar xjf $INDIR/boost_1_55_0.tar.bz2
tar --warning=no-timestamp -xjf $INDIR/boost_1_55_0.tar.bz2
cd boost_1_55_0
GCCVERSION=$($HOST-g++ -E -dM $(mktemp --suffix=.h) | grep __VERSION__ | cut -d ' ' -f 3 | cut -d '"' -f 2)
echo "using gcc : $GCCVERSION : $HOST-g++
@ -68,12 +71,16 @@ script: | @@ -68,12 +71,16 @@ script: |
# http://statmt.org/~s0565741/software/boost_1_52_0/libs/context/doc/html/context/requirements.html
# "For cross-compiling the lib you must specify certain additional properties at bjam command line: target-os, abi, binary-format, architecture and address-model."
./bjam toolset=gcc binary-format=pe target-os=windows threadapi=win32 address-model=$BITS threading=multi variant=release link=static runtime-link=static --user-config=user-config.jam --without-mpi --without-python -sNO_BZIP2=1 -sNO_ZLIB=1 --layout=tagged --build-type=complete --prefix="$INSTALLPREFIX" $MAKEOPTS install
# post-process all generated libraries to be deterministic
# extract them to a temporary directory then re-build them deterministically
for LIB in $(find $INSTALLPREFIX -name \*.a); do
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
rm $LIB
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
done
#
cd "$INSTALLPREFIX"
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
zip -r $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip *
unset LD_PRELOAD
unset FAKETIME
find | sort | zip -X@ $OUTDIR/boost-win$BITS-1.55.0-gitian-r6.zip
done # for BITS in

15
contrib/gitian-descriptors/deps-win.yml

@ -26,6 +26,7 @@ script: | @@ -26,6 +26,7 @@ script: |
export FAKETIME=$REFERENCE_DATETIME
export TZ=UTC
INDIR=$HOME/build
TEMPDIR=$HOME/tmp
# Input Integrity Check
echo "f74f15e8c8ff11aa3d5bb5f276d202ec18d7246e95f961db76054199c69c1ae3 openssl-1.0.1e.tar.gz" | sha256sum -c
echo "12edc0df75bf9abd7f82f821795bcee50f42cb2e5f76a6a281b85732798364ef db-4.8.30.NC.tar.gz" | sha256sum -c
@ -107,11 +108,21 @@ script: | @@ -107,11 +108,21 @@ script: |
#
tar xjf $INDIR/qrencode-3.4.3.tar.bz2
cd qrencode-3.4.3
png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST --enable-static --disable-shared --without-tools --disable-maintainer-mode --disable-dependency-tracking
png_CFLAGS="-I$INSTALLPREFIX/include" png_LIBS="-L$INSTALLPREFIX/lib -lpng" ./configure --prefix=$INSTALLPREFIX --host=$HOST --enable-static --disable-shared --without-tools --disable-dependency-tracking
# Workaround to prevent re-configuring by make (resulting in missing m4 error); make all files have a date in the past
find . -print0 | xargs -r0 touch -t 200001010000
make
make install
cd ..
# post-process all generated libraries to be deterministic
# extract them to a temporary directory then re-build them deterministically
for LIB in $(find $INSTALLPREFIX -name \*.a); do
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
rm $LIB
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
done
#
cd $INSTALLPREFIX
zip -r $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip include lib
find include lib | sort | zip -X@ $OUTDIR/bitcoin-deps-win$BITS-gitian-r10.zip
done # for BITS in

19
contrib/gitian-descriptors/gitian-win.yml

@ -35,6 +35,7 @@ script: | @@ -35,6 +35,7 @@ script: |
export TZ=UTC
INDIR=$HOME/build
OPTFLAGS='-O2'
TEMPDIR="$HOME/tempdir"
NEEDDIST=1
# Qt: workaround for determinism in resource ordering
# Qt5's rcc uses a QHash to store the files for the resource.
@ -54,6 +55,7 @@ script: | @@ -54,6 +55,7 @@ script: |
else
HOST=x86_64-w64-mingw32
fi
export PATH=$STAGING/host/bin:$PATH
mkdir -p $STAGING $BUILDDIR $BINDIR
#
cd $STAGING
@ -63,22 +65,18 @@ script: | @@ -63,22 +65,18 @@ script: |
unzip $INDIR/protobuf-win${BITS}-2.5.0-gitian-r4.zip
if [ "$NEEDDIST" == "1" ]; then
# Make source code archive which is architecture independent so it only needs to be done once
cd $HOME/build/
cd bitcoin
export PATH=$STAGING/host/bin:$PATH
export TAR_OPTIONS=--mtime=`echo $REFERENCE_DATETIME | awk '{ print $1 }'`
cd $HOME/build/bitcoin
./autogen.sh
./configure --bindir=$OUTDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}"
make dist
mkdir -p $OUTDIR/src
cp -f bitcoin-*.tar.* $OUTDIR/src
DISTNAME=`echo bitcoin-*.tar.gz`
NEEDDIST=0
fi
# Build platform-dependent executables from source archive
cd $BUILDDIR
mkdir -p distsrc
cd distsrc
tar --strip-components=1 -xf $HOME/build/bitcoin/bitcoin-*.tar.*
tar --strip-components=1 -xf $HOME/build/bitcoin/$DISTNAME
./configure --bindir=$BINDIR --prefix=$STAGING --host=$HOST --with-qt-plugindir=$STAGING/plugins --with-qt-incdir=$STAGING/include --with-qt-bindir=$STAGING/host/bin --with-boost=$STAGING --disable-maintainer-mode --with-protoc-bindir=$STAGING/host/bin --disable-dependency-tracking CPPFLAGS="-I$STAGING/include ${OPTFLAGS}" LDFLAGS="-L$STAGING/lib ${OPTFLAGS}" CXXFLAGS="-frandom-seed=bitcoin ${OPTFLAGS}"
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
@ -90,3 +88,10 @@ script: | @@ -90,3 +88,10 @@ script: |
unset FAKETIME
done # for BITS in
# sort distribution tar file and normalize user/group/mtime information for deterministic output
mkdir -p $OUTDIR/src
rm -rf $TEMPDIR
mkdir -p $TEMPDIR
cd $TEMPDIR
tar -xvf $HOME/build/bitcoin/$DISTNAME | sort | tar --no-recursion -cT /dev/stdin --mode='u+rw,go+r-w,a+X' --owner=0 --group=0 --mtime="$REFERENCE_DATETIME" | gzip -n > $OUTDIR/src/$DISTNAME

15
contrib/gitian-descriptors/protobuf-win.yml

@ -17,6 +17,7 @@ script: | @@ -17,6 +17,7 @@ script: |
#
export TZ=UTC
INDIR=$HOME/build
TEMPDIR=$HOME/tmp
OPTFLAGS="-O2"
# Integrity Check
echo "13bfc5ae543cf3aa180ac2485c0bc89495e3ae711fc6fab4f8ffe90dfb4bb677 protobuf-2.5.0.tar.bz2" | sha256sum -c
@ -44,13 +45,21 @@ script: | @@ -44,13 +45,21 @@ script: |
# Now recompile with the mingw cross-compiler:
make distclean
./configure --prefix=$INSTALLPREFIX --enable-shared=no --disable-dependency-tracking --with-protoc=$INSTALLPREFIX/host/bin/protoc --host=$HOST CXXFLAGS="-frandom-seed=11 ${OPTFLAGS}"
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
make
make install
# post-process all generated libraries to be deterministic
# extract them to a temporary directory then re-build them deterministically
for LIB in $(find $INSTALLPREFIX -name \*.a); do
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
rm $LIB
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
done
#
cd $INSTALLPREFIX
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
zip -r $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip include lib host
find include lib host | sort | zip -X@ $OUTDIR/protobuf-win$BITS-2.5.0-gitian-r4.zip
unset LD_PRELOAD
unset FAKETIME
done # for BITS in

33
contrib/gitian-descriptors/qt-win.yml

@ -21,6 +21,7 @@ script: | @@ -21,6 +21,7 @@ script: |
# Defines
export TZ=UTC
INDIR=$HOME/build
TEMPDIR=$HOME/tmp
# Qt: workaround for determinism in resource ordering
# Qt5's rcc uses a QHash to store the files for the resource.
# A security fix in QHash makes the ordering of keys to be different on every run
@ -55,31 +56,37 @@ script: | @@ -55,31 +56,37 @@ script: |
cd qt-everywhere-opensource-src-5.2.0
SPECNAME="win32-g++"
SPECFILE="qtbase/mkspecs/${SPECNAME}/qmake.conf"
sed 's/$TODAY/2011-01-30/' -i configure
sed 's/qt_instdate=`date +%Y-%m-%d`/qt_instdate=2011-01-30/' -i qtbase/configure
sed --posix "s|QMAKE_CFLAGS = -pipe -fno-keep-inline-dllexport|QMAKE_CFLAGS\t\t= -pipe -fno-keep-inline-dllexport -isystem /usr/$HOST/include/ -frandom-seed=qtbuild -I$DEPSDIR/include|" -i ${SPECFILE}
sed --posix "s|QMAKE_LFLAGS =|QMAKE_LFLAGS\t\t= -L$DEPSDIR/lib|" -i ${SPECFILE}
# ar adds timestamps to every object file included in the static library
# providing -D as ar argument is supposed to solve it, but doesn't work as qmake strips off the arguments and adds -M to pass a script...
# which somehow cannot be combined with other flags.
# use faketime only for ar, as it confuses make/qmake into hanging sometimes
sed --posix "s|QMAKE_LIB = \\\$\\\${CROSS_COMPILE}ar -ru|QMAKE_LIB\t\t= $HOME/ar -Dr|" -i ${SPECFILE}
echo '#!/bin/bash' > $HOME/ar
echo 'export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1' >> $HOME/ar
echo "$HOST-ar \"\$@\"" >> $HOME/ar
chmod +x $HOME/ar
# Before we tried to pass arguments to ar (static linking) in using QMAKE_LIB, however
# qt removes the arguments for ar and provides a script which makes it impossible to pass the determinism flag -
# so rather than try to replace ar, post-process all libraries and plugins at the end.
#
# Don't load faketime while compiling Qt, qmake will get stuck in nearly infinite loops
#export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
export FAKETIME=$REFERENCE_DATETIME
#export FAKETIME=$REFERENCE_DATETIME
#
# Compile static libraries, and use statically linked openssl (-openssl-linked):
OPENSSL_LIBS="-L$DEPSDIR/lib -lssl -lcrypto -lgdi32" ./configure -prefix $INSTALLPREFIX -bindir $INSTALLPREFIX/host/bin -confirm-license -release -opensource -static -xplatform $SPECNAME -device-option CROSS_COMPILE="$HOST-" -no-audio-backend -no-javascript-jit -no-sql-sqlite -no-sql-odbc -no-nis -no-cups -no-iconv -no-dbus -no-gif -no-opengl -no-compile-examples -no-feature-style-windowsce -no-feature-style-windowsmobile -no-qml-debug -openssl-linked -skip qtsvg -skip qtwebkit -skip qtwebkit-examples -skip qtserialport -skip qtdeclarative -skip qtmultimedia -skip qtimageformats -skip qtlocation -skip qtsensors -skip qtquick1 -skip qtquickcontrols -skip qtactiveqt -skip qtconnectivity -skip qtwinextras -skip qtxmlpatterns -skip qtscript -skip qtdoc -system-libpng -system-zlib
make $MAKEOPTS install
# post-process all generated libraries and plugins to be deterministic
# extract them to a temporary directory then re-build them deterministically
for LIB in $(find $INSTALLPREFIX -name *.a); do
rm -rf $TEMPDIR && mkdir $TEMPDIR && cd $TEMPDIR
$HOST-ar xv $LIB | cut -b5- > /tmp/list.txt
rm $LIB
$HOST-ar crsD $LIB $(cat /tmp/list.txt)
done
#
cd $INSTALLPREFIX
# Remove unused non-deterministic stuff
rm host/bin/qtpaths.exe lib/libQt5Bootstrap.a lib/libQt5Bootstrap.la
# as zip stores file timestamps, use faketime to intercept stat calls to set dates for all files to reference date
export LD_PRELOAD=/usr/lib/faketime/libfaketime.so.1
zip -r $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip *
export FAKETIME=$REFERENCE_DATETIME
find -print0 | xargs -r0 touch # fix up timestamps before packaging
find | sort | zip -X@ $OUTDIR/qt-win${BITS}-5.2.0-gitian-r2.zip
unset LD_PRELOAD
unset FAKETIME
done # for BITS in

Loading…
Cancel
Save