mirror of https://github.com/YGGverse/YGGo.git
phpyggdrasilcrawlermysqljs-lessspideralt-websphinxsearch-engineopen-sourcedistributedwebcurlparserfts5privacy-orientedsphinxsearchfederativeweb-archivepdo
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
72 lines
2.7 KiB
72 lines
2.7 KiB
2 years ago
|
source common
|
||
2 years ago
|
{
|
||
|
type = mysql
|
||
|
|
||
|
sql_host = localhost
|
||
|
sql_user =
|
||
|
sql_pass =
|
||
|
sql_db =
|
||
|
sql_port = 3306 # optional, default is 3306
|
||
2 years ago
|
}
|
||
2 years ago
|
|
||
2 years ago
|
source hostPage : common
|
||
|
{
|
||
2 years ago
|
sql_query = \
|
||
2 years ago
|
SELECT `hostPage`.`hostPageId`, \
|
||
|
`hostPage`.`uri`, \
|
||
1 year ago
|
REPLACE(REPLACE(REPLACE(REPLACE(`hostPage`.`uri`, '/', ' '), '_', ' '), '-', ' '), '.', ' ') AS `hostPageURIKeywords`, \
|
||
1 year ago
|
`hostPage`.`rank`, \
|
||
2 years ago
|
`host`.`name`, \
|
||
1 year ago
|
IF (`host`.`port` IS NOT NULL, \
|
||
1 year ago
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`), \
|
||
1 year ago
|
CONCAT(`host`.`scheme`, '://', `host`.`name`)), \
|
||
|
CRC32 (IF (`host`.`port` IS NOT NULL, \
|
||
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`), \
|
||
|
CONCAT(`host`.`scheme`, '://', `host`.`name`))), \
|
||
1 year ago
|
IF (`host`.`port` IS NOT NULL, \
|
||
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`), \
|
||
1 year ago
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`)), \
|
||
|
CRC32 (IF (`host`.`port` IS NOT NULL, \
|
||
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, ':', `host`.`port`, `hostPage`.`uri`), \
|
||
|
CONCAT(`host`.`scheme`, '://', `host`.`name`, `hostPage`.`uri`))), \
|
||
2 years ago
|
REGEXP_REPLACE(`hostPage`.`mime`, '^([A-z-]+)/[A-z-]+.*', '$1') AS `mime`, \
|
||
1 year ago
|
(SELECT GROUP_CONCAT(`hostPageDescription`.`title`) \
|
||
2 years ago
|
FROM `hostPageDescription` \
|
||
1 year ago
|
WHERE `hostPageDescription`.`hostPageId` = `hostPage`.`hostPageId`) AS `title`, \
|
||
|
(SELECT GROUP_CONCAT(`hostPageDescription`.`description`) \
|
||
|
FROM `hostPageDescription` \
|
||
|
WHERE `hostPageDescription`.`hostPageId` = `hostPage`.`hostPageId`) AS `description`, \
|
||
|
(SELECT GROUP_CONCAT(`hostPageDescription`.`keywords`) \
|
||
|
FROM `hostPageDescription` \
|
||
|
WHERE `hostPageDescription`.`hostPageId` = `hostPage`.`hostPageId`) AS `keywords` \
|
||
2 years ago
|
FROM `hostPage` \
|
||
|
JOIN `host` ON (`host`.`hostId` = `hostPage`.`hostId`) \
|
||
1 year ago
|
WHERE `hostPage`.`httpCode` = 200 AND `hostPage`.`timeBanned` IS NULL AND `hostPage`.`mime` IS NOT NULL \
|
||
2 years ago
|
|
||
2 years ago
|
sql_attr_uint = rank
|
||
|
sql_attr_string = mime
|
||
2 years ago
|
}
|
||
|
|
||
2 years ago
|
index hostPage
|
||
|
{
|
||
2 years ago
|
source = hostPage
|
||
2 years ago
|
morphology = stem_cz, stem_ar, lemmatize_de_all, lemmatize_ru_all, lemmatize_en_all # stem_enru
|
||
2 years ago
|
path = /var/lib/sphinxsearch/data/hostPage
|
||
|
|
||
|
min_word_len = 2
|
||
|
min_prefix_len = 2
|
||
|
|
||
|
html_strip = 1
|
||
|
|
||
|
index_exact_words = 1
|
||
2 years ago
|
}
|
||
|
|
||
|
indexer
|
||
|
{
|
||
2 years ago
|
mem_limit = 256M
|
||
|
lemmatizer_cache = 256M
|
||
2 years ago
|
}
|
||
|
|
||
|
common {
|
||
|
lemmatizer_base = /var/lib/sphinxsearch/dicts # http://sphinxsearch.com/downloads/dicts
|
||
2 years ago
|
}
|