mirror of
https://github.com/YGGverse/Yo.git
synced 2025-03-13 05:41:18 +00:00
624 lines
17 KiB
PHP
624 lines
17 KiB
PHP
<?php
|
|
|
|
// Prevent multi-thread execution
|
|
$semaphore = sem_get(
|
|
crc32(
|
|
__DIR__ . '.yo.cli.document.crawl'
|
|
),
|
|
1
|
|
);
|
|
|
|
if (false === sem_acquire($semaphore, true))
|
|
{
|
|
exit ('process execution locked by another thread!' . PHP_EOL);
|
|
}
|
|
|
|
// Load dependencies
|
|
require_once __DIR__ . '/../../../vendor/autoload.php';
|
|
|
|
// Init config
|
|
$config = json_decode(
|
|
file_get_contents(
|
|
__DIR__ . '/../../../config.json'
|
|
)
|
|
);
|
|
|
|
// Set global options
|
|
define(
|
|
'CONFIG_CLI_DOCUMENT_CRAWL_CURL_DOWNLOAD_SIZE_MAX',
|
|
$config->cli->document->crawl->curl->download->size->max
|
|
);
|
|
|
|
// Init client
|
|
$client = new \Manticoresearch\Client(
|
|
[
|
|
'host' => $config->manticore->server->host,
|
|
'port' => $config->manticore->server->port,
|
|
]
|
|
);
|
|
|
|
// Init search
|
|
$search = new \Manticoresearch\Search(
|
|
$client
|
|
);
|
|
|
|
$search->setIndex(
|
|
$config->manticore->index->document->name
|
|
);
|
|
|
|
$search->match(
|
|
'*',
|
|
'url'
|
|
);
|
|
|
|
$search->sort(
|
|
'time',
|
|
'asc'
|
|
);
|
|
|
|
$search->sort( // randomize new pages queue
|
|
'RAND()'
|
|
);
|
|
|
|
$search->limit(
|
|
$config->cli->document->crawl->queue->limit
|
|
);
|
|
|
|
// Init index
|
|
$index = $client->index(
|
|
$config->manticore->index->document->name
|
|
);
|
|
|
|
// Begin queue
|
|
foreach($search->get() as $document)
|
|
{
|
|
// Define data
|
|
$time = time();
|
|
|
|
$data =
|
|
[
|
|
'url' => $document->get('url'),
|
|
'crc32url' => $document->get('crc32url'),
|
|
'title' => $document->get('title'),
|
|
'description' => $document->get('description'),
|
|
'keywords' => $document->get('keywords'),
|
|
'code' => $document->get('code'),
|
|
'size' => $document->get('size'),
|
|
'mime' => $document->get('mime'),
|
|
'time' => $time
|
|
];
|
|
|
|
// Debug target
|
|
echo sprintf(
|
|
'index "%s" in "%s"' . PHP_EOL,
|
|
$document->get('url'),
|
|
$config->manticore->index->document->name
|
|
);
|
|
|
|
// Update index time anyway and set reset code to 404
|
|
$index->updateDocument(
|
|
[
|
|
'time' => time(),
|
|
'code' => 404
|
|
],
|
|
$document->getId()
|
|
);
|
|
|
|
// Request remote URL
|
|
$request = curl_init(
|
|
$document->get('url')
|
|
);
|
|
|
|
// Drop URL with long response
|
|
curl_setopt(
|
|
$request,
|
|
CURLOPT_CONNECTTIMEOUT,
|
|
$config->cli->document->crawl->curl->connection->timeout
|
|
);
|
|
|
|
curl_setopt(
|
|
$request,
|
|
CURLOPT_TIMEOUT,
|
|
$config->cli->document->crawl->curl->connection->timeout
|
|
);
|
|
|
|
// Prevent huge content download e.g. media streams URL
|
|
curl_setopt(
|
|
$request,
|
|
CURLOPT_RETURNTRANSFER,
|
|
true
|
|
);
|
|
|
|
curl_setopt(
|
|
$request,
|
|
CURLOPT_NOPROGRESS,
|
|
false
|
|
);
|
|
|
|
curl_setopt(
|
|
$request,
|
|
CURLOPT_PROGRESSFUNCTION,
|
|
function(
|
|
$download,
|
|
$downloaded,
|
|
$upload,
|
|
$uploaded
|
|
) {
|
|
return $downloaded > CONFIG_CLI_DOCUMENT_CRAWL_CURL_DOWNLOAD_SIZE_MAX ? 1 : 0;
|
|
}
|
|
);
|
|
|
|
// Begin request
|
|
if ($response = curl_exec($request))
|
|
{
|
|
// Update HTTP code or skip on empty
|
|
if ($code = curl_getinfo($request, CURLINFO_HTTP_CODE))
|
|
{
|
|
$data['code'] = $code;
|
|
|
|
} else continue;
|
|
|
|
// Update size or skip on empty
|
|
if ($size = curl_getinfo($request, CURLINFO_SIZE_DOWNLOAD))
|
|
{
|
|
$data['size'] = $size;
|
|
|
|
} else continue;
|
|
|
|
// Update MIME type or skip on empty
|
|
if ($mime = curl_getinfo($request, CURLINFO_CONTENT_TYPE))
|
|
{
|
|
$data['mime'] = $mime;
|
|
|
|
} else continue;
|
|
|
|
// DOM crawler
|
|
if (false !== stripos($mime, 'text/html'))
|
|
{
|
|
$crawler = new Symfony\Component\DomCrawler\Crawler();
|
|
$crawler->addHtmlContent(
|
|
$response
|
|
);
|
|
|
|
// Get title
|
|
foreach ($crawler->filter('head > title')->each(function($node) {
|
|
|
|
return $node->text();
|
|
|
|
}) as $value)
|
|
{
|
|
if (!empty($value))
|
|
{
|
|
$data['title'] = html_entity_decode(
|
|
$value
|
|
);
|
|
}
|
|
}
|
|
|
|
// Get description
|
|
foreach ($crawler->filter('head > meta[name="description"]')->each(function($node) {
|
|
|
|
return $node->attr('content');
|
|
|
|
}) as $value)
|
|
{
|
|
if (!empty($value))
|
|
{
|
|
$data['description'] = html_entity_decode(
|
|
$value
|
|
);
|
|
}
|
|
}
|
|
|
|
// Get keywords
|
|
$keywords = '';
|
|
foreach ($crawler->filter('head > meta[name="keywords"]')->each(function($node) {
|
|
|
|
return $node->attr('content');
|
|
|
|
}) as $value)
|
|
{
|
|
if (!empty($value))
|
|
{
|
|
$data['keywords'] = html_entity_decode(
|
|
$value
|
|
);
|
|
}
|
|
}
|
|
|
|
echo sprintf(
|
|
'index "%s" updated: %s %s' . PHP_EOL,
|
|
$config->manticore->index->document->name,
|
|
print_r(
|
|
$result,
|
|
true
|
|
),
|
|
print_r(
|
|
$data,
|
|
true
|
|
),
|
|
);
|
|
|
|
// Crawl documents
|
|
$documents = [];
|
|
|
|
$scheme = parse_url($document->get('url'), PHP_URL_SCHEME);
|
|
$host = parse_url($document->get('url'), PHP_URL_HOST);
|
|
$port = parse_url($document->get('url'), PHP_URL_PORT);
|
|
|
|
foreach ($config->cli->document->crawl->selector as $selector => $settings)
|
|
{
|
|
foreach ($crawler->filter($selector)->each(function($node) {
|
|
|
|
return $node;
|
|
|
|
}) as $value) {
|
|
|
|
if ($url = $value->attr($settings->attribute))
|
|
{
|
|
//Make relative links absolute
|
|
if (!parse_url($url, PHP_URL_HOST))
|
|
{
|
|
$url = $scheme . '://' . $host . ($port ? ':' . $port : null) .
|
|
'/' .
|
|
trim(
|
|
ltrim(
|
|
str_replace(
|
|
[
|
|
'./',
|
|
'../'
|
|
],
|
|
'',
|
|
$url
|
|
),
|
|
'/'
|
|
),
|
|
'.'
|
|
);
|
|
}
|
|
|
|
// Regex rules
|
|
if (!preg_match($settings->regex, $url))
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// External host rules
|
|
if (!$settings->external && parse_url($url, PHP_URL_HOST) != $host)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
$documents[] = $url;
|
|
}
|
|
}
|
|
}
|
|
|
|
if ($documents)
|
|
{
|
|
foreach (array_unique($documents) as $url)
|
|
{
|
|
// Apply stripos condition
|
|
$skip = false;
|
|
|
|
foreach ($config->cli->document->crawl->skip->stripos->url as $condition)
|
|
{
|
|
if (false !== stripos($url, $condition)) {
|
|
|
|
$skip = true;
|
|
|
|
break;
|
|
}
|
|
}
|
|
|
|
if ($skip)
|
|
{
|
|
echo sprintf(
|
|
'skip "%s" by stripos condition "%s"' . PHP_EOL,
|
|
$url,
|
|
print_r(
|
|
$config->cli->document->crawl->skip->stripos->url,
|
|
true
|
|
)
|
|
);
|
|
|
|
continue;
|
|
}
|
|
|
|
// Save index
|
|
$url = trim($url);
|
|
$crc32url = crc32($url);
|
|
|
|
if (!$index->search('')
|
|
->filter('crc32url', $crc32url)
|
|
->limit(1)
|
|
->get()
|
|
->getTotal())
|
|
{
|
|
$index->addDocument(
|
|
[
|
|
'url' => $url,
|
|
'crc32url' => $crc32url
|
|
]
|
|
);
|
|
|
|
echo sprintf(
|
|
'add "%s" to "%s"' . PHP_EOL,
|
|
$url,
|
|
$config->manticore->index->document->name
|
|
);
|
|
}
|
|
}
|
|
}
|
|
}
|
|
|
|
// Replace document data
|
|
// https://github.com/manticoresoftware/manticoresearch-php/issues/10#issuecomment-612685916
|
|
$result = $index->replaceDocument(
|
|
$data,
|
|
$document->getId()
|
|
);
|
|
|
|
// Create snap
|
|
if ($config->cli->document->crawl->snap->enabled && $code === 200)
|
|
{
|
|
try
|
|
{
|
|
// Generate path
|
|
$time = time();
|
|
|
|
$md5url = md5(
|
|
$document->get('url')
|
|
);
|
|
|
|
/// absolute
|
|
if ('/' === substr($config->snap->storage->tmp->directory, 0, 1))
|
|
{
|
|
$filepath = $config->snap->storage->tmp->directory;
|
|
}
|
|
|
|
/// relative
|
|
else
|
|
{
|
|
$filepath = __DIR__ . '/../../../' . $config->snap->storage->tmp->directory;
|
|
}
|
|
|
|
$filepath = sprintf(
|
|
'%s/%s',
|
|
$filepath,
|
|
implode(
|
|
'/',
|
|
str_split(
|
|
$md5url
|
|
)
|
|
)
|
|
);
|
|
|
|
@mkdir($filepath, 0755, true);
|
|
|
|
$tmp = sprintf(
|
|
'%s/%s.tar',
|
|
$filepath,
|
|
$time
|
|
);
|
|
|
|
// Compress response to archive
|
|
$snap = new PharData($tmp);
|
|
|
|
$snap->addFromString(
|
|
'DATA',
|
|
$response
|
|
);
|
|
|
|
$snap->addFromString(
|
|
'MIME',
|
|
$mime
|
|
);
|
|
|
|
$snap->addFromString(
|
|
'URL',
|
|
$document->get('url')
|
|
);
|
|
|
|
$snap->compress(
|
|
Phar::GZ
|
|
);
|
|
|
|
unlink( // remove tarball
|
|
$tmp
|
|
);
|
|
|
|
$tmp = sprintf(
|
|
'%s.gz',
|
|
$tmp
|
|
);
|
|
|
|
// Copy to local storage on enabled
|
|
if ($config->snap->storage->local->enabled)
|
|
{
|
|
$allowed = false;
|
|
|
|
// Check for mime allowed
|
|
foreach ($config->snap->storage->local->mime as $whitelist)
|
|
{
|
|
if (false !== stripos($mime, $whitelist))
|
|
{
|
|
$allowed = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Check size limits
|
|
if ($size > $config->snap->storage->local->size->max)
|
|
{
|
|
$allowed = false;
|
|
}
|
|
|
|
// Copy snap to the permanent storage
|
|
if ($allowed)
|
|
{
|
|
/// absolute
|
|
if ('/' === substr($config->snap->storage->local->directory, 0, 1))
|
|
{
|
|
$filepath = $config->snap->storage->local->directory;
|
|
}
|
|
|
|
/// relative
|
|
else
|
|
{
|
|
$filepath = __DIR__ . '/../../../' . $config->snap->storage->local->directory;
|
|
}
|
|
|
|
$filepath = sprintf(
|
|
'%s/%s',
|
|
$filepath,
|
|
implode(
|
|
'/',
|
|
str_split(
|
|
$md5url
|
|
)
|
|
)
|
|
);
|
|
|
|
@mkdir($filepath, 0755, true);
|
|
|
|
$filename = sprintf(
|
|
'%s/%s',
|
|
$filepath,
|
|
basename(
|
|
$tmp
|
|
)
|
|
);
|
|
|
|
copy(
|
|
$tmp,
|
|
$filename
|
|
);
|
|
}
|
|
}
|
|
|
|
// Copy to FTP storage on enabled
|
|
foreach ($config->snap->storage->remote->ftp as $ftp)
|
|
{
|
|
// Resource enabled
|
|
if (!$ftp->enabled)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
$allowed = false;
|
|
|
|
// Check for mime allowed
|
|
foreach ($ftp->mime as $whitelist)
|
|
{
|
|
if (false !== stripos($mime, $whitelist))
|
|
{
|
|
$allowed = true;
|
|
break;
|
|
}
|
|
}
|
|
|
|
// Check size limits
|
|
if ($size > $ftp->size->max)
|
|
{
|
|
$allowed = false;
|
|
}
|
|
|
|
if (!$allowed)
|
|
{
|
|
continue;
|
|
}
|
|
|
|
// Prepare location
|
|
$filepath = implode(
|
|
'/',
|
|
str_split(
|
|
$md5url
|
|
)
|
|
);
|
|
|
|
$filename = sprintf(
|
|
'%s/%s',
|
|
$filepath,
|
|
basename(
|
|
$tmp
|
|
)
|
|
);
|
|
|
|
// Init connection
|
|
$attempt = 1;
|
|
|
|
do {
|
|
|
|
$remote = new \Yggverse\Ftp\Client();
|
|
|
|
$connection = $remote->connect(
|
|
$ftp->connection->host,
|
|
$ftp->connection->port,
|
|
$ftp->connection->username,
|
|
$ftp->connection->password,
|
|
$ftp->connection->directory,
|
|
$ftp->connection->timeout,
|
|
$ftp->connection->passive
|
|
);
|
|
|
|
// Remote host connected
|
|
if ($connection) {
|
|
|
|
$remote->mkdir(
|
|
$filepath,
|
|
true
|
|
);
|
|
|
|
$remote->copy(
|
|
$tmp,
|
|
$filename
|
|
);
|
|
|
|
$remote->close();
|
|
|
|
// On remote connection lost, repeat attempt
|
|
} else {
|
|
|
|
// Stop connection attempts on limit provided
|
|
if ($ftp->connection->attempts->limit > 0 && $attempt > $ftp->connection->attempts->limit)
|
|
{
|
|
break;
|
|
}
|
|
|
|
// Log event
|
|
echo sprintf(
|
|
_('[attempt: %s] wait for remote storage "%s" reconnection...') . PHP_EOL,
|
|
$attempt++,
|
|
$ftp->connection->host,
|
|
);
|
|
|
|
// Delay next attempt
|
|
sleep(
|
|
$ftp->connection->attempts->delay
|
|
);
|
|
}
|
|
|
|
} while ($connection === false);
|
|
}
|
|
|
|
// Remove tmp data
|
|
@unlink(
|
|
$tmp
|
|
);
|
|
}
|
|
|
|
catch (Exception $exception)
|
|
{
|
|
var_dump(
|
|
$exception
|
|
);
|
|
}
|
|
}
|
|
}
|
|
|
|
// Crawl queue delay
|
|
sleep(
|
|
$config->cli->document->crawl->queue->limit
|
|
);
|
|
} |