YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-01-27 06:54:21 +00:00
Code Issues Projects Releases Wiki Activity

add collision events debug

Browse Source
This commit is contained in:
yggverse 2024-03-27 04:27:49 +02:00
parent afe0db48e3
commit 27564c4fbc
1 changed files with 46 additions and 9 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

55
src/cli/document/crawl.php
View File

@ -603,19 +603,22 @@ foreach($index->search('')
}
// Save index
$url = trim($url);
$crc32url = crc32($url);
$url = trim(
$url
);
if (!$index->search('')
->filter('id', $crc32url)
->limit(1)
->get()
->getTotal())
$crc32url = crc32(
$url
);
// Check url does not registered yet
$results = $index->search('')->filter('id', $crc32url)->get();
if (!$results->getTotal())
{
$index->addDocument(
[
'url' => $url
'url' => $url
],
$crc32url
);
@ -630,6 +633,40 @@ foreach($index->search('')
);
}
}
// URL already exists
else
{
// Print notice level notice
if ($config->cli->document->crawl->debug->level->notice)
{
echo sprintf(
_('[%s] [notice] URL "%s" already registered with CRC32 "%d"') . PHP_EOL,
date('c'),
$url,
$crc32url
);
}
// Check for event details
foreach ($results as $result)
{
// Is collision
if ($url != $result->get('url'))
{
if ($config->cli->document->crawl->debug->level->warning)
{
echo sprintf(
_('[%s] [warning] ID "%d" collision for target url "%s" stored as "%s"') . PHP_EOL,
date('c'),
$crc32url,
$url,
$result->get('url')
);
}
}
}
}
}
}
}