Browse Source

add curl dns resolver options #15

main
yggverse 1 month ago
parent
commit
28040c934f
  1. 12
      example/config.json
  2. 46
      src/cli/document/crawl.php

12
example/config.json

@ -177,6 +177,18 @@ @@ -177,6 +177,18 @@
{
"max":10485760
}
},
"dns":
{
"cache":
{
"timeout":120,
"global":false
},
"shuffle":false,
"interface":null,
"ip4address":null,
"ip6address":null
}
},
"queue":

46
src/cli/document/crawl.php

@ -189,6 +189,52 @@ foreach($index->search('') @@ -189,6 +189,52 @@ foreach($index->search('')
$document->get('url')
);
// DNS options
curl_setopt(
$request,
CURLOPT_DNS_CACHE_TIMEOUT,
$config->cli->document->crawl->curl->dns->cache->timeout
);
curl_setopt(
$request,
CURLOPT_DNS_USE_GLOBAL_CACHE,
$config->cli->document->crawl->curl->dns->cache->global
);
curl_setopt(
$request,
CURLOPT_DNS_SHUFFLE_ADDRESSES,
$config->cli->document->crawl->curl->dns->shuffle
);
if ($config->cli->document->crawl->curl->dns->interface)
{
curl_setopt(
$request,
CURLOPT_DNS_INTERFACE,
$config->cli->document->crawl->curl->dns->interface
);
}
if ($config->cli->document->crawl->curl->dns->ip4address)
{
curl_setopt(
$request,
CURLOPT_DNS_LOCAL_IP4,
$config->cli->document->crawl->curl->dns->ip4address
);
}
if ($config->cli->document->crawl->curl->dns->ip6address)
{
curl_setopt(
$request,
CURLOPT_DNS_LOCAL_IP6,
$config->cli->document->crawl->curl->dns->ip6address
);
}
// Drop URL with long response
curl_setopt(
$request,

Loading…
Cancel
Save