|
|
|
@ -189,6 +189,52 @@ foreach($index->search('')
@@ -189,6 +189,52 @@ foreach($index->search('')
|
|
|
|
|
$document->get('url') |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
// DNS options |
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_CACHE_TIMEOUT, |
|
|
|
|
$config->cli->document->crawl->curl->dns->cache->timeout |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_USE_GLOBAL_CACHE, |
|
|
|
|
$config->cli->document->crawl->curl->dns->cache->global |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_SHUFFLE_ADDRESSES, |
|
|
|
|
$config->cli->document->crawl->curl->dns->shuffle |
|
|
|
|
); |
|
|
|
|
|
|
|
|
|
if ($config->cli->document->crawl->curl->dns->interface) |
|
|
|
|
{ |
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_INTERFACE, |
|
|
|
|
$config->cli->document->crawl->curl->dns->interface |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if ($config->cli->document->crawl->curl->dns->ip4address) |
|
|
|
|
{ |
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_LOCAL_IP4, |
|
|
|
|
$config->cli->document->crawl->curl->dns->ip4address |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
if ($config->cli->document->crawl->curl->dns->ip6address) |
|
|
|
|
{ |
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|
CURLOPT_DNS_LOCAL_IP6, |
|
|
|
|
$config->cli->document->crawl->curl->dns->ip6address |
|
|
|
|
); |
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
// Drop URL with long response |
|
|
|
|
curl_setopt( |
|
|
|
|
$request, |
|
|
|
|