Browse Source

update yggverse/net version

main
yggverse 9 months ago
parent
commit
982d1f1246
  1. 4
      composer.json
  2. 24
      src/cli/document/crawl.php

4
composer.json

@ -20,7 +20,7 @@ @@ -20,7 +20,7 @@
"jdenticon/jdenticon": "^1.0",
"yggverse/ftp": "^1.0",
"gregwar/captcha": "^1.2",
"yggverse/net": "^1.2",
"yggverse/yo-tools": "^0.1.0"
"yggverse/net": "^1.3",
"yggverse/yo-tools": "dev-main"
}
}

24
src/cli/document/crawl.php

@ -498,15 +498,23 @@ foreach($index->search('') @@ -498,15 +498,23 @@ foreach($index->search('')
if ($url = $value->attr($settings->attribute))
{
//Make relative links absolute
$url = \Yggverse\YoTools\Link::relative2absolute(
$document->get('url'),
$url,
$scheme,
$host,
$port,
// Convert relative links to absolute
$address = new \Yggverse\Net\Address(
$url
);
if ($address->isRelative())
{
$base = new \Yggverse\Net\Address(
$document->get('url')
);
if ($absolute = $address->getAbsolute($base))
{
$url = $absolute;
}
}
// Regex rules
if (!preg_match($settings->regex, $url))
{
@ -514,7 +522,7 @@ foreach($index->search('') @@ -514,7 +522,7 @@ foreach($index->search('')
}
// External host rules
if (!$settings->external && parse_url($url, PHP_URL_HOST) != $host)
if (!$settings->external && parse_url($url, PHP_URL_HOST) != $address->getHost())
{
continue;
}

Loading…
Cancel
Save