kvazar/crawler
1
0
Fork 0
mirror of https://github.com/kvazar-network/crawler.git synced 2025-01-22 04:45:18 +00:00
Code Issues Projects Releases Wiki Activity

skip base64 index

Browse Source
This commit is contained in:
ghost 2024-02-13 14:43:09 +02:00
parent 0acf603eac
commit 1420cb9750
1 changed files with 11 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

12
src/index.php
View File

@ -341,7 +341,7 @@ for ($block = $state; $block <= $blocks; $block++)
);
}
// Skip binary content
// Skip binary index
if (false === mb_detect_encoding((string) $namespace, null, true)
||
false === mb_detect_encoding((string) $key, null, true)
@ -351,6 +351,16 @@ for ($block = $state; $block <= $blocks; $block++)
continue;
}
// Skip base64 index
if ((strlen($namespace) % 4 == 0 && base64_encode(base64_decode($namespace, true)) === $namespace)
||
(strlen($key) % 4 == 0 && base64_encode(base64_decode($key, true)) === $key)
||
(strlen($value) % 4 == 0 && base64_encode(base64_decode($value, true)) === $value))
{
continue;
}
// Add index record
$index->add(
$raw['time'],