mirror of
https://github.com/YGGverse/Yo.git
synced 2025-01-30 08:24:16 +00:00
make custom resolver optionally required to continue the crawl #15
This commit is contained in:
parent
1b8bcb084a
commit
bc9fc470e6
@ -138,6 +138,7 @@
|
|||||||
"resolver":
|
"resolver":
|
||||||
{
|
{
|
||||||
"enabled":true,
|
"enabled":true,
|
||||||
|
"require":false,
|
||||||
"providers":
|
"providers":
|
||||||
[
|
[
|
||||||
"1.1.1.1",
|
"1.1.1.1",
|
||||||
|
@ -295,25 +295,49 @@ foreach($index->search('')
|
|||||||
|
|
||||||
else
|
else
|
||||||
{
|
{
|
||||||
// Log event
|
// Custom resolver required to continue
|
||||||
if ($config->cli->document->crawl->debug->level->warning)
|
if ($config->cli->document->crawl->resolver->require)
|
||||||
{
|
{
|
||||||
echo sprintf(
|
// Log event
|
||||||
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL,
|
if ($config->cli->document->crawl->debug->level->warning)
|
||||||
date('c'),
|
{
|
||||||
$base->getHost(),
|
echo sprintf(
|
||||||
$attempt++,
|
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL,
|
||||||
print_r(
|
date('c'),
|
||||||
$errors,
|
$base->getHost(),
|
||||||
true
|
$attempt++,
|
||||||
)
|
print_r(
|
||||||
|
$errors,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
// Next connection delay
|
||||||
|
sleep(
|
||||||
|
$config->cli->document->crawl->resolver->connection->delay
|
||||||
);
|
);
|
||||||
}
|
}
|
||||||
|
|
||||||
// Next connection delay
|
// Use system-wide resolver, break the pending loop
|
||||||
sleep(
|
else
|
||||||
$config->cli->document->crawl->resolver->connection->delay
|
{
|
||||||
);
|
if ($config->cli->document->crawl->debug->level->warning)
|
||||||
|
{
|
||||||
|
echo sprintf(
|
||||||
|
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), use system-wide...') . PHP_EOL,
|
||||||
|
date('c'),
|
||||||
|
$base->getHost(),
|
||||||
|
$attempt++,
|
||||||
|
print_r(
|
||||||
|
$errors,
|
||||||
|
true
|
||||||
|
)
|
||||||
|
);
|
||||||
|
}
|
||||||
|
|
||||||
|
break;
|
||||||
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
} while (!$resolved);
|
} while (!$resolved);
|
||||||
|
Loading…
x
Reference in New Issue
Block a user