Browse Source

add script tag support

main
ghost 1 year ago
parent
commit
1f33205236
  1. 19
      crontab/crawler.php

19
crontab/crawler.php

@ -954,6 +954,25 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND @@ -954,6 +954,25 @@ foreach ($db->getHostPageCrawlQueue(CRAWL_PAGE_LIMIT, time() - CRAWL_PAGE_SECOND
];
}
foreach (@$dom->getElementsByTagName('script') as $script) {
// Skip nodes without href attribute
if (!$src = @$script->getAttribute('src')) {
continue;
}
// Add link to queue
$links[] = [
'title' => null,
'description' => null,
'keywords' => null,
'data' => null,
'mime' => null,
'ref' => $src,
];
}
foreach (@$dom->getElementsByTagName('link') as $link) {
// Skip nodes without href attribute

Loading…
Cancel
Save