|
|
@ -104,12 +104,16 @@ if (API_ENABLED) { |
|
|
|
'WEBSITE_DOMAIN' => WEBSITE_DOMAIN, |
|
|
|
'WEBSITE_DOMAIN' => WEBSITE_DOMAIN, |
|
|
|
|
|
|
|
|
|
|
|
'CRAWL_URL_REGEXP' => CRAWL_URL_REGEXP, |
|
|
|
'CRAWL_URL_REGEXP' => CRAWL_URL_REGEXP, |
|
|
|
|
|
|
|
|
|
|
|
'CRAWL_HOST_DEFAULT_PAGES_LIMIT' => CRAWL_HOST_DEFAULT_PAGES_LIMIT, |
|
|
|
'CRAWL_HOST_DEFAULT_PAGES_LIMIT' => CRAWL_HOST_DEFAULT_PAGES_LIMIT, |
|
|
|
'CRAWL_HOST_DEFAULT_STATUS' => CRAWL_HOST_DEFAULT_STATUS, |
|
|
|
'CRAWL_HOST_DEFAULT_STATUS' => CRAWL_HOST_DEFAULT_STATUS, |
|
|
|
'CRAWL_HOST_DEFAULT_META_ONLY' => CRAWL_HOST_DEFAULT_META_ONLY, |
|
|
|
'CRAWL_HOST_DEFAULT_META_ONLY' => CRAWL_HOST_DEFAULT_META_ONLY, |
|
|
|
|
|
|
|
'CLEAN_HOST_SECONDS_OFFSET' => CLEAN_HOST_SECONDS_OFFSET, |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
'CRAWL_PAGE_SECONDS_OFFSET' => CRAWL_PAGE_SECONDS_OFFSET, |
|
|
|
|
|
|
|
|
|
|
|
'CRAWL_ROBOTS_DEFAULT_RULES' => CRAWL_ROBOTS_DEFAULT_RULES, |
|
|
|
'CRAWL_ROBOTS_DEFAULT_RULES' => CRAWL_ROBOTS_DEFAULT_RULES, |
|
|
|
'CRAWL_ROBOTS_POSTFIX_RULES' => CRAWL_ROBOTS_POSTFIX_RULES, |
|
|
|
'CRAWL_ROBOTS_POSTFIX_RULES' => CRAWL_ROBOTS_POSTFIX_RULES, |
|
|
|
'CLEAN_HOST_SECONDS_OFFSET' => CLEAN_HOST_SECONDS_OFFSET, |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
'API_VERSION' => API_VERSION, |
|
|
|
'API_VERSION' => API_VERSION, |
|
|
|
|
|
|
|
|
|
|
|