YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-02-03 10:25:52 +00:00
Code Issues Projects Releases Wiki Activity

add CRAWL_META_ONLY option

Browse Source
This commit is contained in:
ghost 2023-04-03 03:07:54 +03:00
parent dc55dcb9b5
commit 79663c84db
2 changed files with 3 additions and 1 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
config/app.php.txt
View File

@ -22,3 +22,5 @@ define('CRAWL_PAGE_LIMIT', 10);
define('CRAWL_PAGE_SECONDS_OFFSET', 3600); define('CRAWL_PAGE_SECONDS_OFFSET', 3600);
define('CRAWL_URL_REGEXP', '/^.*$/ui'); define('CRAWL_URL_REGEXP', '/^.*$/ui');
define('CRAWL_META_ONLY', false);

2
crontab/crawler.php
View File

@ -67,7 +67,7 @@ foreach ($db->getPageQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET)
Filter::pageTitle($title->item(0)->nodeValue), Filter::pageTitle($title->item(0)->nodeValue),
Filter::pageDescription($description), Filter::pageDescription($description),
Filter::pageKeywords($keywords), Filter::pageKeywords($keywords),
Filter::pageData($url->getContent()), CRAWL_META_ONLY ? '' : Filter::pageData($content),
time()); time());
// Update images // Update images