YGGverse
/
YGGo
mirror of https://github.com/YGGverse/YGGo.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

add mime content type crawling #1

main
ghost 2 years ago
parent
0bd95d7f4d
commit
702a14b634
6 changed files with 87 additions and 70 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      README.md
  2. 97
      crontab/crawler.php
  3. BIN
      database/yggo.mwb
  4. 7
      library/filter.php
  5. 41
      library/mysql.php
  6. 10
      public/search.php

2
README.md
Unescape View File

@ -175,7 +175,7 @@ GET m=SphinxQL @@ -175,7 +175,7 @@ GET m=SphinxQL
* [x] Auto stop crawling on disk quota reached
* [x] Transactions support to prevent data loss on queue failures
* [x] Distributed index crawling between YGGo nodes trough manifest API
* [ ] MIME Content-type crawler settings
* [x] MIME Content-type crawler settings
* [ ] Indexing new sites homepage in higher priority
* [ ] Redirect codes extended processing
* [ ] Palette image index / filter

97
crontab/crawler.php
Unescape View File

@ -236,40 +236,39 @@ try { @@ -236,40 +236,39 @@ try {
continue;
}
// Save image content on data settings enabled
if (!CRAWL_HOST_DEFAULT_META_ONLY) {
// Skip image processing on MIME type not provided
if (!$hostImageContentType = $curl->getContentType()) {
// Skip image processing on MIME type not provided
if (!$contentType = $curl->getContentType()) {
continue;
}
// Skip image processing on MIME type not allowed in settings
if (false === strpos($contentType, CRAWL_IMAGE_MIME_TYPE)) {
continue;
}
continue;
}
// Skip image processing on MIME type not allowed in settings
if (false === strpos($hostImageContentType, CRAWL_IMAGE_MIME_TYPE)) {
// Skip image processing without returned content
if (!$content = $curl->getContent()) {
continue;
}
continue;
}
// Skip image processing without returned content
if (!$content = $curl->getContent()) {
// Convert remote image data to base64 string to prevent direct URL call
if (!$hostImageType = @pathinfo($queueHostImageURL, PATHINFO_EXTENSION)) {
continue;
}
continue;
}
// Convert remote image data to base64 string to prevent direct URL call
if (!$hostImageExtension = @pathinfo($queueHostImageURL, PATHINFO_EXTENSION)) {
if (!$hostImageBase64 = @base64_encode($curl->getContent())) {
continue;
}
continue;
}
if (!$hostImageBase64 = @base64_encode($curl->getContent())) {
$hostImagesIndexed += $db->updateHostImageData($hostImage->hostImageId, (string) 'data:image/' . $hostImageType . ';base64,' . $hostImageBase64, time());
continue;
}
$hostImagesIndexed += $db->updateHostImage($hostImage->hostImageId,
Filter::mime($hostImageContentType),
(!CRAWL_HOST_DEFAULT_META_ONLY ? 'data:image/' . $hostImageExtension . ';base64,' . $hostImageBase64 : null),
time());
}
// Process pages crawl queue
@ -344,12 +343,25 @@ try { @@ -344,12 +343,25 @@ try {
}
}
// Append page with meta robots:noindex value to the robotsPostfix disallow list
if (false !== stripos($metaRobots, 'noindex')) {
continue;
}
// Skip page links following by robots:nofollow attribute detected
if (false !== stripos($metaRobots, 'nofollow')) {
continue;
}
// Update queued page data
$hostPagesIndexed += $db->updateHostPage($queueHostPage->hostPageId,
Filter::pageTitle($title->item(0)->nodeValue),
Filter::pageDescription($metaDescription),
Filter::pageKeywords($metaKeywords),
CRAWL_HOST_DEFAULT_META_ONLY ? null : Filter::pageData($content));
Filter::pageTitle($title->item(0)->nodeValue),
Filter::pageDescription($metaDescription),
Filter::pageKeywords($metaKeywords),
Filter::mime($contentType),
CRAWL_HOST_DEFAULT_META_ONLY ? null : Filter::pageData($content));
// Update manifest registry
if (CRAWL_MANIFEST && !empty($metaYggoManifest) && filter_var($metaYggoManifest, FILTER_VALIDATE_URL) && preg_match(CRAWL_URL_REGEXP, $metaYggoManifest)) {
@ -357,25 +369,13 @@ try { @@ -357,25 +369,13 @@ try {
$metaYggoManifestCRC32 = crc32($metaYggoManifest);
if (!$db->getManifest($metaYggoManifestCRC32)) {
$db->addManifest($metaYggoManifestCRC32,
$db->addManifest($metaYggoManifestCRC32,
$metaYggoManifest,
(string) CRAWL_MANIFEST_DEFAULT_STATUS,
time());
}
}
// Append page with meta robots:noindex value to the robotsPostfix disallow list
if (false !== stripos($metaRobots, 'noindex')) {
continue;
}
// Skip page links following by robots:nofollow attribute detected
if (false !== stripos($metaRobots, 'nofollow')) {
continue;
}
// Collect page images
if (CRAWL_HOST_DEFAULT_IMAGES_LIMIT > 0) {
@ -402,7 +402,7 @@ try { @@ -402,7 +402,7 @@ try {
$imageSrc = $queueHostPage->scheme . '://' .
$queueHostPage->name .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
($queueHostPage->port ? ':' . $queueHostPage->port : '') .
'/' . trim(ltrim(str_replace(['./', '../'], '', $imageSrc), '/'), '.');
}
@ -466,16 +466,19 @@ try { @@ -466,16 +466,19 @@ try {
// Init robots parser
$robots = new Robots(($hostRobots ? (string) $hostRobots : (string) CRAWL_ROBOTS_DEFAULT_RULES) . PHP_EOL . ($hostRobotsPostfix ? (string) $hostRobotsPostfix : (string) CRAWL_ROBOTS_POSTFIX_RULES));
// Save image info
// Save new image info
$hostImageId = $db->getHostImageId($hostId, crc32($hostImageURI->string));
if (!$hostImageId && // image not exists
$hostStatus && // host enabled
$robots->uriAllowed($hostImageURI->string) && // src allowed by robots.txt rules
$hostImageLimit > $db->getTotalHostImages($hostId)) { // images quantity not reached host limit
$hostStatus && // host enabled
$robots->uriAllowed($hostImageURI->string) && // src allowed by robots.txt rules
$hostImageLimit > $db->getTotalHostImages($hostId)) { // images quantity not reached host limit
// Add host image
if ($hostImageId = $db->addHostImage($hostId, crc32($hostImageURI->string), $hostImageURI->string, time(), null, 200)) {
if ($hostImageId = $db->addHostImage($hostId,
crc32($hostImageURI->string),
$hostImageURI->string,
time())) {
$hostImagesAdded++;

BIN
database/yggo.mwb
View File

Binary file not shown.

7
library/filter.php
Unescape View File

@ -9,6 +9,13 @@ class Filter { @@ -9,6 +9,13 @@ class Filter {
return trim(urldecode($url));
}
static public function mime(mixed $mime) {
$mime = (string) $mime;
return trim($mime);
}
static public function pageTitle(mixed $title) {
$title = (string) $title;

41
library/mysql.php
Unescape View File

@ -180,13 +180,14 @@ class MySQL { @@ -180,13 +180,14 @@ class MySQL {
}
public function addHostImage(int $hostId,
int $crc32uri,
string $uri,
int $timeAdded,
mixed $timeUpdated = null,
mixed $httpCode = null,
mixed $rank = null,
mixed $data = null) {
int $crc32uri,
string $uri,
int $timeAdded,
mixed $timeUpdated = null,
mixed $httpCode = null,
mixed $mime = null,
mixed $rank = null,
mixed $data = null) {
$query = $this->_db->prepare('INSERT INTO `hostImage` ( `hostId`,
`crc32uri`,
@ -194,10 +195,11 @@ class MySQL { @@ -194,10 +195,11 @@ class MySQL {
`timeAdded`,
`timeUpdated`,
`httpCode`,
`mime`,
`rank`,
`data`) VALUES (?, ?, ?, ?, ?, ?, ?, ?)');
`data`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?)');
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $httpCode, $rank, $data]);
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $httpCode, $mime, $rank, $data]);
return $this->_db->lastInsertId();
}
@ -224,13 +226,14 @@ class MySQL { @@ -224,13 +226,14 @@ class MySQL {
return $query->rowCount();
}
public function updateHostImageData(int $hostImageId,
string $data,
int $timeUpdated) {
public function updateHostImage(int $hostImageId,
string $mime,
mixed $data,
int $timeUpdated) {
$query = $this->_db->prepare('UPDATE `hostImage` SET `data` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1');
$query = $this->_db->prepare('UPDATE `hostImage` SET `mime` = ?, `data` = ?, `timeUpdated` = ? WHERE `hostImageId` = ? LIMIT 1');
$query->execute([$data, $timeUpdated, $hostImageId]);
$query->execute([$mime, $data, $timeUpdated, $hostImageId]);
return $query->rowCount();
}
@ -439,6 +442,7 @@ class MySQL { @@ -439,6 +442,7 @@ class MySQL {
int $timeAdded,
mixed $timeUpdated = null,
mixed $httpCode = null,
mixed $mime = null,
mixed $rank = null,
mixed $metaTitle = null,
mixed $metaDescription = null,
@ -451,13 +455,14 @@ class MySQL { @@ -451,13 +455,14 @@ class MySQL {
`timeAdded`,
`timeUpdated`,
`httpCode`,
`mime`,
`rank`,
`metaTitle`,
`metaDescription`,
`metaKeywords`,
`data`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
`data`) VALUES (?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?, ?)');
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $httpCode, $rank, $metaTitle, $metaDescription, $metaKeywords, $data]);
$query->execute([$hostId, $crc32uri, $uri, $timeAdded, $timeUpdated, $httpCode, $mime, $rank, $metaTitle, $metaDescription, $metaKeywords, $data]);
return $this->_db->lastInsertId();
}
@ -466,14 +471,16 @@ class MySQL { @@ -466,14 +471,16 @@ class MySQL {
mixed $metaTitle,
mixed $metaDescription,
mixed $metaKeywords,
string $mime,
mixed $data) {
$query = $this->_db->prepare('UPDATE `hostPage` SET `metaTitle` = ?,
`metaDescription` = ?,
`metaKeywords` = ?,
`mime` = ?,
`data` = ? WHERE `hostPageId` = ? LIMIT 1');
$query->execute([$metaTitle, $metaDescription, $metaKeywords, $data, $hostPageId]);
$query->execute([$metaTitle, $metaDescription, $metaKeywords, $mime, $data, $hostPageId]);
return $query->rowCount();
}

10
public/search.php
Unescape View File

@ -353,17 +353,17 @@ if (!empty($q)) { @@ -353,17 +353,17 @@ if (!empty($q)) {
$db->updateHostImageHttpCode($hostImage->hostImageId, (int) $hostImageHttpCode, time());
if (200 != $hostImageHttpCode) continue;
if (!$hostImageContentType = $hostImageCurl->getContentType()) continue;
if (false === strpos($hostImageContentType, CRAWL_IMAGE_MIME_TYPE)) continue;
// Convert remote image data to base64 string to prevent direct URL call
if (!$hostImageType = @pathinfo($hostImageURL, PATHINFO_EXTENSION)) continue;
if (!$hostImageExtension = @pathinfo($hostImageURL, PATHINFO_EXTENSION)) continue;
if (!$hostImageBase64 = @base64_encode($hostImageCurl->getContent())) continue;
$hostImageURLencoded = 'data:image/' . $hostImageType . ';base64,' . $hostImageBase64;
$hostImageURLencoded = 'data:image/' . $hostImageExtension . ';base64,' . $hostImageBase64;
// Save image content on data settings enabled
if (!CRAWL_HOST_DEFAULT_META_ONLY) {
$db->updateHostImageData($hostImage->hostImageId, (string) $hostImageURLencoded, time());
}
$db->updateHostImage($hostImage->hostImageId, Filter::mime($hostImageContentType), (!CRAWL_HOST_DEFAULT_META_ONLY ? $hostImageURLencoded : null), time());
// Local image data exists
} else {

Write Preview
Loading…
Cancel
Save