YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-24 21:44:59 +00:00
Code Issues Projects Releases Wiki Activity

fix page/image mime content type detection

Browse Source
This commit is contained in:
ghost 2023-05-08 12:10:57 +03:00
parent 25b6bce2ec
commit bf1eeb332c
1 changed files with 26 additions and 10 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

36
crontab/crawler.php
View File

@ -58,7 +58,7 @@ try {
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest(); $httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload(); $httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime(); $httpRequestsTimeTotal += $curl->getTotalTime();
@ -255,7 +255,7 @@ try {
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest(); $httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload(); $httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime(); $httpRequestsTimeTotal += $curl->getTotalTime();
@ -279,11 +279,19 @@ try {
} }
// Skip image processing on MIME type not allowed in settings // Skip image processing on MIME type not allowed in settings
if (false === strpos(CRAWL_IMAGE_MIME, $hostImageContentType)) { $hostImageBanned = true;
foreach ((array) explode(',', CRAWL_IMAGE_MIME) as $mime) {
if (false !== strpos($hostImageContentType, trim($mime))) {
$hostImageBanned = false;
break;
}
}
if ($hostImageBanned) {
$hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time()); $hostImagesBanned += $db->updateHostImageTimeBanned($queueHostImage->hostImageId, time());
continue;
} }
// Convert remote image data to base64 string // Convert remote image data to base64 string
@ -335,7 +343,7 @@ try {
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest(); $httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload(); $httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime(); $httpRequestsTimeTotal += $curl->getTotalTime();
@ -359,11 +367,19 @@ try {
} }
// Skip page processing on MIME type not allowed in settings // Skip page processing on MIME type not allowed in settings
if (false === strpos(CRAWL_PAGE_MIME, $contentType)) { $hostPageBanned = true;
foreach ((array) explode(',', CRAWL_PAGE_MIME) as $mime) {
if (false !== strpos($contentType, trim($mime))) {
$hostPageBanned = false;
break;
}
}
if ($hostPageBanned) {
$hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time()); $hostPagesBanned += $db->updateHostPageTimeBanned($queueHostPage->hostPageId, time());
continue;
} }
// Skip page processing without returned data // Skip page processing without returned data
@ -669,7 +685,7 @@ try {
// Update curl stats // Update curl stats
$httpRequestsTotal++; $httpRequestsTotal++;
$httpRequestsSizeTotal += $curl->getSizeRequest(); $httpRequestsSizeTotal += $curl->getSizeRequest();
$httpDownloadSizeTotal += $curl->getSizeDownload(); $httpDownloadSizeTotal += $curl->getSizeDownload();
$httpRequestsTimeTotal += $curl->getTotalTime(); $httpRequestsTimeTotal += $curl->getTotalTime();