_rule[$this->_regex(trim($part[1]))] = true; } if (false !== strpos($part[0], 'disallow')) { $this->_rule[$this->_regex(trim($part[1]))] = false; } } } // User-agent * end if ($read && preg_match('!^user-agent:!', $row)) { break; } } } public function indexURL(string $url) { // Unify case match $url = strtolower(trim($url)); // Convert to URI $url = str_replace(parse_url($url, PHP_URL_SCHEME) . '://' . parse_url($url, PHP_URL_HOST) . parse_url($url, PHP_URL_PORT), '', $url); // Index by default $result = true; // Begin index rules by ASC priority foreach ($this->_rule as $rule => $value) { if (preg_match('!^' . $rule . '!', $url)) { $result = $value; } } return $result; } private function _regex(string $string) { return str_replace( [ '*', '?' ], [ '.*', '\?' ], $string ); } }