YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-03-13 05:41:18 +00:00
Code Issues Projects Releases Wiki Activity

enable cleanup on configuration update, delete snap match cleanup conditions

Browse Source
This commit is contained in:
yggverse 2024-03-21 18:23:07 +02:00
parent 25cfd4fac1
commit e635ec6dc9
1 changed files with 73 additions and 14 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

87
src/cli/document/clean.php
View File

@ -37,10 +37,6 @@ $index = $client->index(
);
// Apply new configuration rules
/*
@TODO this case removes some not relevant records, the solution still wanted:
https://github.com/manticoresoftware/manticoresearch-php/discussions/196
echo _('apply new configuration rules...') . PHP_EOL;
foreach ($config->cli->document->crawl->skip->stripos->url as $condition)
@ -50,23 +46,86 @@ foreach ($config->cli->document->crawl->skip->stripos->url as $condition)
$condition
);
$result = $index->deleteDocuments(
[
'match' =>
[
'url' => $condition
]
]
// Process conditions match indexer settings
if (
isset($config->manticore->index->document->settings->min_word_len)
&&
mb_strlen($condition) < $config->manticore->index->document->settings->min_word_len
) {
echo sprintf(
_('condition skipped as "min_word_len" value is "%d"') . PHP_EOL,
$config->manticore->index->document->settings->min_word_len
);
continue;
}
if (
isset($config->manticore->index->document->settings->min_prefix_len)
&&
mb_strlen($condition) < $config->manticore->index->document->settings->min_prefix_len
) {
echo sprintf(
_('condition skipped as "min_prefix_len" value is "%d"') . PHP_EOL,
$config->manticore->index->document->settings->min_prefix_len
);
continue;
}
// Begin search query
$documents = 0;
$snaps = 0;
foreach($index->search(sprintf('@url "%s"', $condition))->get() as $document)
{
// Delete found document by it ID
$result = $index->deleteDocument(
$document->getId()
);
// Delete local snaps
$location = sprintf(
'%s/%s',
'/' === substr($config->snap->storage->local->directory, 0, 1) ? $config->snap->storage->local->directory
: __DIR__ . '/../../../' . $config->snap->storage->local->directory,
implode(
'/',
str_split(
md5(
$document->get('url')
)
)
)
);
if (file_exists($location))
{
if (unlink($location))
{
$snaps++;
}
}
$documents++;
// @TODO delete remote snaps
}
echo sprintf(
_('deleted documents: %d') . PHP_EOL,
$documents
);
echo sprintf(
_('documents deleted: %d') . PHP_EOL,
$result['deleted']
_('deleted local snaps: %d') . PHP_EOL,
$snaps
);
}
echo _('new configuration rules apply completed.') . PHP_EOL;
*/
// Optimize indexes
echo _('indexes optimization begin...') . PHP_EOL;