make custom resolver optionally required to continue the crawl #15

This commit is contained in:
yggverse 2024-04-07 04:57:02 +03:00
parent 1b8bcb084a
commit bc9fc470e6
2 changed files with 40 additions and 15 deletions

View File

@ -138,6 +138,7 @@
"resolver": "resolver":
{ {
"enabled":true, "enabled":true,
"require":false,
"providers": "providers":
[ [
"1.1.1.1", "1.1.1.1",

View File

@ -295,25 +295,49 @@ foreach($index->search('')
else else
{ {
// Log event // Custom resolver required to continue
if ($config->cli->document->crawl->debug->level->warning) if ($config->cli->document->crawl->resolver->require)
{ {
echo sprintf( // Log event
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL, if ($config->cli->document->crawl->debug->level->warning)
date('c'), {
$base->getHost(), echo sprintf(
$attempt++, _('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL,
print_r( date('c'),
$errors, $base->getHost(),
true $attempt++,
) print_r(
$errors,
true
)
);
}
// Next connection delay
sleep(
$config->cli->document->crawl->resolver->connection->delay
); );
} }
// Next connection delay // Use system-wide resolver, break the pending loop
sleep( else
$config->cli->document->crawl->resolver->connection->delay {
); if ($config->cli->document->crawl->debug->level->warning)
{
echo sprintf(
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), use system-wide...') . PHP_EOL,
date('c'),
$base->getHost(),
$attempt++,
print_r(
$errors,
true
)
);
}
break;
}
} }
} while (!$resolved); } while (!$resolved);