Merge pull request #4752

df7565d depends: add sensible download timeout/retry values (Cory Fields)
c897b1e depends: add a fallback path in case package sources go missing (Cory Fields)

depends/Makefile

@@ -6,12 +6,15 @@ SDK_PATH ?= $(BASEDIR)/SDKs
 NO_QT ?=
 NO_WALLET ?=
 NO_UPNP ?=
+FALLBACK_DOWNLOAD_PATH ?= https://bitcoincore.org/depends-sources
 
 BUILD = $(shell ./config.guess)
 HOST ?= $(BUILD)
 PATCHES_PATH = $(BASEDIR)/patches
 BASEDIR = $(CURDIR)
 HASH_LENGTH:=11
+DOWNLOAD_CONNECT_TIMEOUT:=10
+DOWNLOAD_RETRIES:=3
 
 host:=$(BUILD)
 ifneq ($(HOST),)

depends/README.usage

@@ -18,6 +18,7 @@ The following can be set when running make: make FOO=bar
 SOURCES_PATH: downloaded sources will be placed here
 BASE_CACHE: built packages will be placed here
 SDK_PATH: Path where sdk's can be found (used by OSX)
+FALLBACK_DOWNLOAD_PATH: If a source file can't be fetched, try here before giving up
 NO_QT: Don't download/build/cache qt and its dependencies
 NO_WALLET: Don't download/build/cache libs needed to enable the wallet
 NO_UPNP: Don't download/build/cache packages needed for enabling upnp

depends/builders/darwin.mk

@@ -7,7 +7,7 @@ build_darwin_OTOOL: = $(shell xcrun -f otool)
 build_darwin_NM: = $(shell xcrun -f nm)
 build_darwin_INSTALL_NAME_TOOL:=$(shell xcrun -f install_name_tool)
 build_darwin_SHA256SUM = shasum -a 256
-build_darwin_DOWNLOAD = curl -L -o
+build_darwin_DOWNLOAD = curl --connect-timeout $(DOWNLOAD_CONNECT_TIMEOUT) --retry $(DOWNLOAD_RETRIES) -L -o
 
 #darwin host on darwin builder. overrides darwin host preferences.
 darwin_CC=$(shell xcrun -f clang) -mmacosx-version-min=$(OSX_MIN_VERSION)

depends/builders/linux.mk

@@ -1,2 +1,2 @@
 build_linux_SHA256SUM = sha256sum
-build_linux_DOWNLOAD = wget -nv -O
+build_linux_DOWNLOAD = wget --timeout=$(DOWNLOAD_CONNECT_TIMEOUT) --tries=$(DOWNLOAD_RETRIES) -nv -O

depends/funcs.mk

@@ -20,11 +20,13 @@ $(sort $(foreach dep,$(2),$(2) $(call int_get_all_dependencies,$(1),$($(dep)_dep
 endef
 
 define fetch_file
-(test -f $(SOURCES_PATH)/$(3) || \
-  ( mkdir -p $$($(1)_extract_dir) && $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(2)" && \
-    echo "$(4)  $$($(1)_extract_dir)/$(3).temp" > $$($(1)_extract_dir)/.$(3).hash && \
-    $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$(3).hash && \
-    mv $$($(1)_extract_dir)/$(3).temp $(SOURCES_PATH)/$(3) ))
+(test -f $(SOURCES_PATH)/$(4) || \
+  ( mkdir -p $$($(1)_extract_dir) && \
+  ( $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(2)/$(3)" || \
+    $(build_DOWNLOAD) "$$($(1)_extract_dir)/$(3).temp" "$(FALLBACK_DOWNLOAD_PATH)/$(3)" ) && \
+    echo "$(5)  $$($(1)_extract_dir)/$(4).temp" > $$($(1)_extract_dir)/.$(4).hash && \
+    $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$(4).hash && \
+    mv $$($(1)_extract_dir)/$(4).temp $(SOURCES_PATH)/$(4) ))
 endef
 
 define int_get_build_recipe_hash
@@ -65,7 +67,7 @@ $(1)_download_path_fixed=$(subst :,\:,$$($(1)_download_path))
 
 
 #default commands
-$(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)/$$($(1)_download_file)),$($(1)_file_name),$($(1)_sha256_hash))
+$(1)_fetch_cmds ?= $(call fetch_file,$(1),$(subst \:,:,$$($(1)_download_path_fixed)),$$($(1)_download_file),$($(1)_file_name),$($(1)_sha256_hash))
 $(1)_extract_cmds ?= mkdir -p $$($(1)_extract_dir) && echo "$$($(1)_sha256_hash)  $$($(1)_source)" > $$($(1)_extract_dir)/.$$($(1)_file_name).hash &&  $(build_SHA256SUM) -c $$($(1)_extract_dir)/.$$($(1)_file_name).hash && tar --strip-components=1 -xf $$($(1)_source)
 $(1)_preprocess_cmds ?=
 $(1)_build_cmds ?=

depends/packages/native_cctools.mk

@@ -23,11 +23,11 @@ $(package)_clang_file_name=clang-llvm-3.2-x86-linux-ubuntu-12.04.tar.gz
 $(package)_clang_sha256_hash=b9d57a88f9514fa1f327a1a703756d0c1c960f4c58494a5bd80313245d13ffff
 
 define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_ld64_download_path)/$($(package)_ld64_download_file),$($(package)_ld64_file_name),$($(package)_ld64_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_dyld_download_path)/$($(package)_dyld_download_file),$($(package)_dyld_file_name),$($(package)_dyld_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_clang_download_path)/$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_toolchain4_download_path)/$($(package)_toolchain4_download_file),$($(package)_toolchain4_file_name),$($(package)_toolchain4_sha256_hash))
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_download_file),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_ld64_download_path),$($(package)_ld64_download_file),$($(package)_ld64_file_name),$($(package)_ld64_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_dyld_download_path),$($(package)_dyld_download_file),$($(package)_dyld_file_name),$($(package)_dyld_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_clang_download_path),$($(package)_clang_download_file),$($(package)_clang_file_name),$($(package)_clang_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_toolchain4_download_path),$($(package)_toolchain4_download_file),$($(package)_toolchain4_file_name),$($(package)_toolchain4_sha256_hash))
 endef
 
 define $(package)_set_vars

depends/packages/native_comparisontool.mk

@@ -15,12 +15,12 @@ $(package)_slf4j-jdk14_file_name=slf4j-jdk14-1.6.4.jar
 $(package)_slf4j-jdk14_sha256_hash=064bd81796710f713f9f4a2309c0e032309934c2d2b4f7d3b6958325e584e13f
 
 define $(package)_fetch_cmds
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_file_name),$($(package)_file_name),$($(package)_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_guava_file_name),$($(package)_guava_file_name),$($(package)_guava_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_h2_file_name),$($(package)_h2_file_name),$($(package)_h2_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_sha256_hash)) && \
-$(call fetch_file,$(package),$($(package)_download_path)/$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_sha256_hash))
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_file_name),$($(package)_file_name),$($(package)_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_guava_file_name),$($(package)_guava_file_name),$($(package)_guava_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_h2_file_name),$($(package)_h2_file_name),$($(package)_h2_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_file_name),$($(package)_sc-light-jdk15on_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_file_name),$($(package)_slf4j-api_sha256_hash)) && \
+$(call fetch_file,$(package),$($(package)_download_path),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_file_name),$($(package)_slf4j-jdk14_sha256_hash))
 endef
 
 define $(package)_extract_cmds