Browse Source

add curl dns resolver options #15

main
yggverse 8 months ago
parent
commit
28040c934f
  1. 12
      example/config.json
  2. 46
      src/cli/document/crawl.php

12
example/config.json

@ -177,6 +177,18 @@
{ {
"max":10485760 "max":10485760
} }
},
"dns":
{
"cache":
{
"timeout":120,
"global":false
},
"shuffle":false,
"interface":null,
"ip4address":null,
"ip6address":null
} }
}, },
"queue": "queue":

46
src/cli/document/crawl.php

@ -189,6 +189,52 @@ foreach($index->search('')
$document->get('url') $document->get('url')
); );
// DNS options
curl_setopt(
$request,
CURLOPT_DNS_CACHE_TIMEOUT,
$config->cli->document->crawl->curl->dns->cache->timeout
);
curl_setopt(
$request,
CURLOPT_DNS_USE_GLOBAL_CACHE,
$config->cli->document->crawl->curl->dns->cache->global
);
curl_setopt(
$request,
CURLOPT_DNS_SHUFFLE_ADDRESSES,
$config->cli->document->crawl->curl->dns->shuffle
);
if ($config->cli->document->crawl->curl->dns->interface)
{
curl_setopt(
$request,
CURLOPT_DNS_INTERFACE,
$config->cli->document->crawl->curl->dns->interface
);
}
if ($config->cli->document->crawl->curl->dns->ip4address)
{
curl_setopt(
$request,
CURLOPT_DNS_LOCAL_IP4,
$config->cli->document->crawl->curl->dns->ip4address
);
}
if ($config->cli->document->crawl->curl->dns->ip6address)
{
curl_setopt(
$request,
CURLOPT_DNS_LOCAL_IP6,
$config->cli->document->crawl->curl->dns->ip6address
);
}
// Drop URL with long response // Drop URL with long response
curl_setopt( curl_setopt(
$request, $request,

Loading…
Cancel
Save