From 28040c934f3daeb5a221ac1c44774c794bba9730 Mon Sep 17 00:00:00 2001 From: yggverse Date: Sat, 6 Apr 2024 08:14:29 +0300 Subject: [PATCH] add curl dns resolver options #15 --- example/config.json | 12 ++++++++++ src/cli/document/crawl.php | 46 ++++++++++++++++++++++++++++++++++++++ 2 files changed, 58 insertions(+) diff --git a/example/config.json b/example/config.json index ec2b1f0..93e9fb4 100644 --- a/example/config.json +++ b/example/config.json @@ -177,6 +177,18 @@ { "max":10485760 } + }, + "dns": + { + "cache": + { + "timeout":120, + "global":false + }, + "shuffle":false, + "interface":null, + "ip4address":null, + "ip6address":null } }, "queue": diff --git a/src/cli/document/crawl.php b/src/cli/document/crawl.php index e0952f2..20d0e4f 100644 --- a/src/cli/document/crawl.php +++ b/src/cli/document/crawl.php @@ -189,6 +189,52 @@ foreach($index->search('') $document->get('url') ); + // DNS options + curl_setopt( + $request, + CURLOPT_DNS_CACHE_TIMEOUT, + $config->cli->document->crawl->curl->dns->cache->timeout + ); + + curl_setopt( + $request, + CURLOPT_DNS_USE_GLOBAL_CACHE, + $config->cli->document->crawl->curl->dns->cache->global + ); + + curl_setopt( + $request, + CURLOPT_DNS_SHUFFLE_ADDRESSES, + $config->cli->document->crawl->curl->dns->shuffle + ); + + if ($config->cli->document->crawl->curl->dns->interface) + { + curl_setopt( + $request, + CURLOPT_DNS_INTERFACE, + $config->cli->document->crawl->curl->dns->interface + ); + } + + if ($config->cli->document->crawl->curl->dns->ip4address) + { + curl_setopt( + $request, + CURLOPT_DNS_LOCAL_IP4, + $config->cli->document->crawl->curl->dns->ip4address + ); + } + + if ($config->cli->document->crawl->curl->dns->ip6address) + { + curl_setopt( + $request, + CURLOPT_DNS_LOCAL_IP6, + $config->cli->document->crawl->curl->dns->ip6address + ); + } + // Drop URL with long response curl_setopt( $request,