diff --git a/library/robots.php b/library/robots.php index 3cbf198..a92b8b6 100644 --- a/library/robots.php +++ b/library/robots.php @@ -16,6 +16,12 @@ class Robots { $row = strtolower(trim($row)); + // User-agent * begin + if (preg_match('!^user-agent:\s?\*!', $row)) { + $read = true; + continue; + } + // Parse sitemap address if (preg_match('!^sitemap:\s?(.*)!', $row, $matches)) { @@ -25,12 +31,6 @@ class Robots { } } - // User-agent * begin - if (preg_match('!^user-agent:\s?\*!', $row)) { - $read = true; - continue; - } - if ($read) { $part = explode(' ', $row);