YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-09-02 17:22:56 +00:00
Code Issues Projects Releases Wiki Activity

fix host page mime detection

Browse Source
This commit is contained in:
ghost 2023-06-13 22:29:28 +03:00
parent c07d6af52f
commit 93c6067fd9
1 changed files with 16 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

32
crontab/crawler.php
View File

@ -449,24 +449,20 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
continue; continue;
} }
// Parse index MIME // Parse MIME
$hostPageIsDom = false; $hostPageIsHtml = false;
$hostPageInMime = false; $hostPageInMime = false;
foreach ((array) explode(',', CRAWL_PAGE_MIME_INDEX) as $mime) { foreach ((array) explode(',', CRAWL_PAGE_MIME_INDEX) as $mime) {
$mime = Filter::mime($mime);
// Check for DOM
if (false !== stripos('text/html', $mime)) {
$hostPageIsDom = true;
$hostPageInMime = true;
break;
}
// Ban page on MIME type not allowed in settings // Ban page on MIME type not allowed in settings
if (false !== stripos(Filter::mime($contentType), $mime)) { if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
// Check for HTML page
if (false !== stripos(Filter::mime($contentType), 'text/html')) {
$hostPageIsHtml = true;
}
$hostPageInMime = true; $hostPageInMime = true;
break; break;
@ -503,7 +499,7 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
$yggoManifest = null; $yggoManifest = null;
// Is DOM content // Is DOM content
if ($hostPageIsDom) { if ($hostPageIsHtml) {
// Parse content // Parse content
$dom = new DomDocument(); $dom = new DomDocument();
@ -598,10 +594,8 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
foreach ((array) explode(',', CRAWL_PAGE_MIME_SNAP_LOCAL) as $mime) { foreach ((array) explode(',', CRAWL_PAGE_MIME_SNAP_LOCAL) as $mime) {
$mime = Filter::mime($mime);
// MIME type allowed in settings // MIME type allowed in settings
if (false !== stripos(Filter::mime($contentType), $mime)) { if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
$snapLocal = true; $snapLocal = true;
break; break;
@ -614,10 +608,8 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
foreach ((array) explode(',', CRAWL_PAGE_MIME_SNAP_MEGA) as $mime) { foreach ((array) explode(',', CRAWL_PAGE_MIME_SNAP_MEGA) as $mime) {
$mime = Filter::mime($mime);
// MIME type allowed in settings // MIME type allowed in settings
if (false !== stripos(Filter::mime($contentType), $mime)) { if (false !== stripos(Filter::mime($contentType), Filter::mime($mime))) {
$snapMega = true; $snapMega = true;
break; break;