YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-09 22:37:57 +00:00
Code Issues Projects Releases Wiki Activity

skip javascript/mailto links index

Browse Source
This commit is contained in:
ghost 2023-04-07 05:19:32 +03:00
parent 2a843449e0
commit 9b9d40a97c
1 changed files with 14 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

14
crontab/crawler.php
View File

@ -100,6 +100,20 @@ foreach ($db->getCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECONDS_OFFSET
continue;
}
// Skip javascript links
if (false !== strpos($href, 'javascript:')) {
continue;
}
// Skip mailto links
if (false !== strpos($href, 'mailto:')) {
continue;
}
// @TODO skip other apps
// Add absolute URL prefixes to the relative links found
if (!parse_url($href, PHP_URL_HOST)) {