YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-02-07 20:34:28 +00:00
Code Issues Projects Releases Wiki Activity

ban host page on encoding not detected

Browse Source
This commit is contained in:
ghost 2023-06-16 13:23:52 +03:00
parent d2469e9adc
commit d96abb8ea8
1 changed files with 12 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

13
crontab/crawler.php
View File

@ -494,7 +494,18 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
// Parse content // Parse content
$dom = new DomDocument(); $dom = new DomDocument();
@$dom->loadHTML(sprintf('<?xml encoding="%s" ?>', mb_detect_encoding($content)) . $content); if ($encoding = mb_detect_encoding($content)) {
@$dom->loadHTML(sprintf('<?xml encoding="%s" ?>', $encoding) . $content);
} else {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
$db->commit();
continue;
}
// Skip index page links without titles // Skip index page links without titles
$title = @$dom->getElementsByTagName('title'); $title = @$dom->getElementsByTagName('title');