YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-01-29 16:04:20 +00:00
Code Issues Projects Releases Wiki Activity

make custom resolver optionally required to continue the crawl #15

Browse Source
This commit is contained in:
yggverse 2024-04-07 04:57:02 +03:00
parent 1b8bcb084a
commit bc9fc470e6
2 changed files with 40 additions and 15 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

1
example/config.json
View File

@ -138,6 +138,7 @@
"resolver":
{
"enabled":true,
"require":false,
"providers":
[
"1.1.1.1",

54
src/cli/document/crawl.php
View File

@ -295,25 +295,49 @@ foreach($index->search('')
else
{
// Log event
if ($config->cli->document->crawl->debug->level->warning)
// Custom resolver required to continue
if ($config->cli->document->crawl->resolver->require)
{
echo sprintf(
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL,
date('c'),
$base->getHost(),
$attempt++,
print_r(
$errors,
true
)
// Log event
if ($config->cli->document->crawl->debug->level->warning)
{
echo sprintf(
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), wait for reconnection...') . PHP_EOL,
date('c'),
$base->getHost(),
$attempt++,
print_r(
$errors,
true
)
);
}
// Next connection delay
sleep(
$config->cli->document->crawl->resolver->connection->delay
);
}
// Next connection delay
sleep(
$config->cli->document->crawl->resolver->connection->delay
);
// Use system-wide resolver, break the pending loop
else
{
if ($config->cli->document->crawl->debug->level->warning)
{
echo sprintf(
_('[%s] [warning] could not resolve "%s" (attempt: %d, response: %s), use system-wide...') . PHP_EOL,
date('c'),
$base->getHost(),
$attempt++,
print_r(
$errors,
true
)
);
}
break;
}
}
} while (!$resolved);