YGGverse
/
YGGo
mirror of https://github.com/YGGverse/YGGo.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

update cli

main
ghost 1 year ago
parent
43776b5ff4
commit
ef170f62f3
1 changed files with 255 additions and 246 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 501
      cli/yggo.php

501
cli/yggo.php
Unescape View File

@ -50,391 +50,384 @@ if (false === sem_acquire($semaphore, true)) { @@ -50,391 +50,384 @@ if (false === sem_acquire($semaphore, true)) {
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
// CLI begin
if (empty($argv[1])) $argv[1] = 'help';
if (!empty($argv[1])) {
switch ($argv[1]) {
switch ($argv[1]) {
case 'crontab':
case 'db':
if (empty($argv[2])) {
if (empty($argv[2])) {
CLI::danger(_('crontab method requires action argument'));
switch ($argv[2]) {
switch ($argv[2]) {
case 'optimize':
case 'crawl':
CLI::notice(_('optimize database tables...'));
CLI::notice(_('crawler queue step begin...'));
$db->optimize();
include_once(__DIR__ . '/../crontab/crawler.php');
CLI::success(_('tables successfully optimized!'));
CLI::notice(_('crawler queue step begin...'));
break;
break;
}
}
case 'clean':
break;
case 'crontab':
CLI::notice(_('cleaner queue step begin...'));
if (empty($argv[2])) {
include_once(__DIR__ . '/../crontab/cleaner.php');
switch ($argv[2]) {
CLI::notice(_('cleaner queue step completed.'));
case 'crawl':
break;
}
}
CLI::notice(_('crawler queue step begin...'));
break;
case 'hostPageSnap':
include_once(__DIR__ . '/../crontab/crawler.php');
if (empty($argv[2])) {
CLI::danger(_('hostPageSnap method requires action argument'));
CLI::break();
exit;
}
CLI::notice(_('crawler queue step begin...'));
break;
switch ($argv[2]) {
case 'clean':
case 'repair':
CLI::notice(_('cleaner queue step begin...'));
// @TODO
CLI::danger(_('this function upgraded but not tested after snaps refactor.'));
CLI::danger(_('make sure you have backups then remove this alert.'));
CLI::break();
exit;
include_once(__DIR__ . '/../crontab/cleaner.php');
// Normalize & cleanup DB
CLI::notice(_('scan database registry for missed snap files...'));
CLI::notice(_('cleaner queue step completed.'));
foreach ($db->getHosts() as $host) {
break;
}
}
foreach ($db->getHostPages($host->hostId) as $hostPage) {
break;
case 'hostPageSnap':
foreach ($db->getHostPageSnaps($hostPage->hostPageId) as $hostPageSnap) {
if (empty($argv[2])) {
// Prepare filenames
$hostPageSnapPath = 'hps/' . substr(trim(chunk_split($hostPageSnap->hostPageSnapId, 1, '/'), '/'), 0, -1);
$hostPageSnapFile = $hostPageSnapPath . substr($hostPageSnap->hostPageSnapId, -1) . '.zip';
switch ($argv[2]) {
// Define variables
$hostPageSnapStorageFilesExists = false;
case 'repair':
// Check file exists
foreach (json_decode(SNAP_STORAGE) as $node => $storages) {
// @TODO
CLI::danger(_('this function upgraded but not tested after snaps refactor.'));
CLI::danger(_('make sure you have backups then remove this alert.'));
CLI::break();
exit;
foreach ($storages as $location => $storage) {
switch ($argv[3]) {
// Generate storage id
$crc32name = crc32(sprintf('%s.%s', $node, $location));
case 'db':
switch ($node) {
// Normalize & cleanup DB
CLI::notice(_('scan database registry for missed snap files...'));
case 'localhost':
foreach ($db->getHosts() as $host) {
// @TODO implemented, not tested
$hostPageSnapFile = $storage->directory . $hostPageSnapFile;
foreach ($db->getHostPages($host->hostId) as $hostPage) {
if (file_exists($hostPageSnapFile)) {
foreach ($db->getHostPageSnaps($hostPage->hostPageId) as $hostPageSnap) {
$hostPageSnapStorageFilesExists = true;
// Prepare filenames
$hostPageSnapPath = 'hps/' . substr(trim(chunk_split($hostPageSnap->hostPageSnapId, 1, '/'), '/'), 0, -1);
$hostPageSnapFile = $hostPageSnapPath . substr($hostPageSnap->hostPageSnapId, -1) . '.zip';
if (!$db->findHostPageSnapStorageByCRC32Name($hostPageSnap->hostPageSnapId, $crc32name)) {
// Define variables
$hostPageSnapStorageFilesExists = false;
if ($db->addHostPageSnapStorage($hostPageSnap->hostPageSnapId, $crc32name, $hostPageSnap->timeAdded)) {
// Check file exists
foreach (json_decode(SNAP_STORAGE) as $node => $storages) {
CLI::warning(sprintf(_('add index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
foreach ($storages as $location => $storage) {
} else {
// Generate storage id
$crc32name = crc32(sprintf('%s.%s', $node, $location));
CLI::success(sprintf(_('skip related index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
}
switch ($node) {
break;
case 'localhost':
case 'ftp':
// @TODO implemented, not tested
$hostPageSnapFile = $storage->directory . $hostPageSnapFile;
$ftp = new Ftp();
if (file_exists($hostPageSnapFile)) {
if ($ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
$hostPageSnapStorageFilesExists = true;
if ($ftp->size($hostPageSnapFile)) {
if (!$db->findHostPageSnapStorageByCRC32Name($hostPageSnap->hostPageSnapId, $crc32name)) {
$hostPageSnapStorageFilesExists = true;
if ($db->addHostPageSnapStorage($hostPageSnap->hostPageSnapId, $crc32name, $hostPageSnap->timeAdded)) {
if (!$db->findHostPageSnapStorageByCRC32Name($hostPageSnap->hostPageSnapId, $crc32name)) {
CLI::warning(sprintf(_('add index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
if ($db->addHostPageSnapStorage($hostPageSnap->hostPageSnapId, $crc32name, $hostPageSnap->timeAdded)) {
} else {
CLI::warning(sprintf(_('add index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
} else {
CLI::success(sprintf(_('skip related index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
}
CLI::success(sprintf(_('skip related index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
}
break;
// Prevent snap deletion from registry on FTP connection lost
} else {
case 'ftp':
CLI::danger(sprintf(_('could not connect to storage %s location %s. operation stopped to prevent the data lose.'), $hostPageSnapStorageName, $location));
CLI::break();
exit;
}
$ftp = new Ftp();
$ftp->close();
if ($ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
break;
}
}
}
if ($ftp->size($hostPageSnapFile)) {
// Files not exists
if (!$hostPageSnapStorageFilesExists) {
$hostPageSnapStorageFilesExists = true;
// Delete snap from registry
try {
if (!$db->findHostPageSnapStorageByCRC32Name($hostPageSnap->hostPageSnapId, $crc32name)) {
$db->beginTransaction();
if ($db->addHostPageSnapStorage($hostPageSnap->hostPageSnapId, $crc32name, $hostPageSnap->timeAdded)) {
foreach ($db->getHostPageSnapStorages($hostPageSnap->hostPageSnapId) as $hostPageSnapStorage) {
CLI::warning(sprintf(_('add index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
} else {
$db->deleteHostPageSnapDownloads($hostPageSnapStorage->hostPageSnapStorageId);
}
CLI::success(sprintf(_('skip related index hostPageSnapId #%s file: %s node: %s location: %s;'), $hostPageSnap->hostPageSnapId, $hostPageSnapFile, $node, $location));
}
}
$db->deleteHostPageSnapStorages($hostPageSnap->hostPageSnapId);
$db->deleteHostPageSnap($hostPageSnap->hostPageSnapId);
// Prevent snap deletion from registry on FTP connection lost
} else {
CLI::warning(sprintf(_('delete hostPageSnapId: #%s timeAdded: %s as not found in file storages;'), $hostPageSnap->hostPageSnapId, $hostPageSnap->timeAdded));
CLI::danger(sprintf(_('could not connect to storage %s location %s. operation stopped to prevent the data lose.'), $hostPageSnapStorageName, $location));
CLI::break();
exit;
}
$db->commit();
$ftp->close();
} catch(Exception $e) {
break;
}
}
}
$db->rollBack();
// Files not exists
if (!$hostPageSnapStorageFilesExists) {
var_dump($e);
}
}
}
}
}
// Delete snap from registry
try {
// Cleanup FS
CLI::notice(_('scan storage for snap files missed in the DB...'));
$db->beginTransaction();
// Copy files to each storage
foreach (json_decode(SNAP_STORAGE) as $node => $storages) {
foreach ($db->getHostPageSnapStorages($hostPageSnap->hostPageSnapId) as $hostPageSnapStorage) {
foreach ($storages as $location => $storage) {
$db->deleteHostPageSnapDownloads($hostPageSnapStorage->hostPageSnapStorageId);
}
// Generate storage id
$crc32name = crc32(sprintf('%s.%s', $node, $location));
$db->deleteHostPageSnapStorages($hostPageSnap->hostPageSnapId);
$db->deleteHostPageSnap($hostPageSnap->hostPageSnapId);
switch ($node) {
CLI::warning(sprintf(_('delete hostPageSnapId: #%s timeAdded: %s as not found in file storages;'), $hostPageSnap->hostPageSnapId, $hostPageSnap->timeAdded));
case 'localhost':
$db->commit();
// @TODO
} catch(Exception $e) {
break;
$db->rollBack();
case 'ftp':
var_dump($e);
}
}
}
}
}
$ftp = new Ftp();
break;
case 'fs':
if ($ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
// Cleanup FS
CLI::notice(_('scan storage for snap files missed in the DB...'));
foreach ($ftp->nlistr($storage->directory) as $filename) {
// Copy files to each storage
foreach (json_decode(SNAP_STORAGE) as $node => $storages) {
if (false !== preg_match(sprintf('!/hps/([\d]+)\.zip$!ui', $storage->directory), $filename, $matches)) {
foreach ($storages as $location => $storage) {
if (!empty($matches[1])) { // hostPageSnapId
// Generate storage id
$crc32name = crc32(sprintf('%s.%s', $node, $location));
if (!$db->getHostPageSnap($matches[1])) {
switch ($node) {
if ($ftp->delete($filename)) {
case 'localhost':
CLI::warning(sprintf(_('delete snap file: #%s from node %s location %s not found in registry;'), $filename, $node, $location));
// @TODO
} else {
break;
CLI::danger(sprintf(_('delete snap file: #%s from node %s location %s not found in registry;'), $filename, $node, $location));
}
case 'ftp':
} else {
$ftp = new Ftp();
CLI::success(sprintf(_('skip snap file: #%s available in node %s location %s;'), $filename, $node, $location));
}
}
}
}
}
if ($ftp->connect($storage->host, $storage->port, $storage->username, $storage->password, $storage->directory, $storage->timeout, $storage->passive)) {
$ftp->close();
foreach ($ftp->nlistr($storage->directory) as $filename) {
break;
}
}
}
if (false !== preg_match(sprintf('!/hps/([\d]+)\.zip$!ui', $storage->directory), $filename, $matches)) {
CLI::success(_('missed snap files successfully deleted!'));
if (!empty($matches[1])) { // hostPageSnapId
// Optimize DB tables
CLI::notice(_('optimize database tables...'));
if (!$db->getHostPageSnap($matches[1])) {
$db->optimize();
if ($ftp->delete($filename)) {
CLI::success(_('tables successfully optimized!'));
CLI::warning(sprintf(_('delete snap file: #%s from node %s location %s not found in registry;'), $filename, $node, $location));
break;
default:
} else {
CLI::danger(_('undefined action argument!'));
}
CLI::danger(sprintf(_('delete snap file: #%s from node %s location %s not found in registry;'), $filename, $node, $location));
}
break;
case 'hostPage':
} else {
if (empty($argv[2])) {
CLI::success(sprintf(_('skip snap file: #%s available in node %s location %s;'), $filename, $node, $location));
}
}
}
}
}
CLI::danger(_('hostPage method requires action argument'));
}
$ftp->close();
switch ($argv[2]) {
break;
}
}
}
case 'rank':
CLI::success(_('missed snap files successfully deleted!'));
break;
}
break;
case 'reindex':
if (empty($argv[3])) {
//@TODO
CLI::danger(_('hostPage rank requires action argument'));
break;
}
}
switch ($argv[3]) {
case 'reindex':
break;
case 'hostPage':
foreach ($db->getHosts() as $host) {
switch ($argv[2]) {
foreach ($db->getHostPages($host->hostId) as $hostPage) {
case 'rank':
$db->updateHostPageRank($hostPage->hostPageId, $db->getTotalExternalHostPageIdSourcesByHostPageIdTarget($hostPage->hostPageId)); // @TODO add library cover
}
}
if (empty($argv[3])) {
CLI::success(_('hostPage rank successfully updated'));
exit;
switch ($argv[3]) {
break;
default:
case 'reindex':
CLI::danger(_('undefined action argument'));
}
foreach ($db->getHosts() as $host) {
break;
case 'truncate':
foreach ($db->getHostPages($host->hostId) as $hostPage) {
$db->truncateHostPageDom();
$db->updateHostPageRank($hostPage->hostPageId, $db->getTotalExternalHostPageIdSourcesByHostPageIdTarget($hostPage->hostPageId)); // @TODO add library cover
}
}
CLI::success(_('hostPageDom table successfully truncated'));
exit;
CLI::success(_('hostPage rank successfully updated'));
exit;
break;
default:
break;
default:
CLI::danger(_('undefined action argument'));
}
CLI::danger(_('undefined action argument'));
}
}
break;
case 'hostPageDom':
break;
}
if (empty($argv[2])) {
break;
case 'hostPageDom':
CLI::danger(_('hostPageDom method requires action argument'));
}
if (empty($argv[2])) {
switch ($argv[2]) {
switch ($argv[2]) {
case 'generate':
case 'generate':
$selectors = [];
$selectors = [];
foreach ((array) explode(';', !empty($argv[3]) ? $argv[3] : (string) CRAWL_HOST_PAGE_DOM_SELECTORS) as $selector) {
foreach ((array) explode(';', !empty($argv[3]) ? $argv[3] : (string) CRAWL_HOST_PAGE_DOM_SELECTORS) as $selector) {
if (!empty($selector)) {
if (!empty($selector)) {
$selectors[] = trim($selector);
}
}
$selectors[] = trim($selector);
}
}
if ($selectors) {
if ($selectors) {
// Init variables
$hostPagesProcessedTotal = 0;
$hostPageDOMAddedTotal = 0;
// Init variables
$hostPagesProcessedTotal = 0;
$hostPageDOMAddedTotal = 0;
// Begin selectors extraction
foreach ($db->getHostPagesByIndexed() as $hostPage) {
// Begin selectors extraction
foreach ($db->getHostPagesByIndexed() as $hostPage) {
if (false !== stripos(Filter::mime($hostPage->mime), 'text/html')) {
if (false !== stripos(Filter::mime($hostPage->mime), 'text/html')) {
if ($hostPageDescription = $db->getLastPageDescription($hostPage->hostPageId)) {
if ($hostPageDescription = $db->getLastPageDescription($hostPage->hostPageId)) {
$hostPagesProcessedTotal++;
$hostPagesProcessedTotal++;
if (!empty($hostPageDescription->data)) {
if (!empty($hostPageDescription->data)) {
$html = str_get_html(base64_decode($hostPageDescription->data));
$html = str_get_html(base64_decode($hostPageDescription->data));
foreach ($selectors as $selector) {
foreach ($selectors as $selector) {
foreach($html->find($selector) as $element) {
foreach($html->find($selector) as $element) {
if (!empty($element->innertext)) {
if (!empty($element->innertext)) {
$hostPageDOMAddedTotal++;
$hostPageDOMAddedTotal++;
$db->addHostPageDom($hostPage->hostPageId,
time(),
$selector,
trim(CRAWL_HOST_PAGE_DOM_STRIP_TAGS ? strip_tags(
preg_replace('/[\s]+/',
' ',
str_replace(['<br />', '<br/>', '<br>', '</'],
[' ', ' ', ' ', ' </'],
$element->innertext))) : $element->innertext));
$db->addHostPageDom($hostPage->hostPageId,
time(),
$selector,
trim(CRAWL_HOST_PAGE_DOM_STRIP_TAGS ? strip_tags(
preg_replace('/[\s]+/',
' ',
str_replace(['<br />', '<br/>', '<br>', '</'],
[' ', ' ', ' ', ' </'],
$element->innertext))) : $element->innertext));
}
}
}
}
}
}
}
}
}
CLI::success(sprintf(_('Host pages processed: %s'), $hostPagesProcessedTotal));
CLI::success(sprintf(_('Host page DOM elements added: %s'), $hostPageDOMAddedTotal));
exit;
}
CLI::danger(_('CRAWL_HOST_PAGE_DOM_SELECTORS not provided in the configuration file'));
exit;
break;
case 'truncate':
CLI::success(sprintf(_('Host pages processed: %s'), $hostPagesProcessedTotal));
CLI::success(sprintf(_('Host page DOM elements added: %s'), $hostPageDOMAddedTotal));
exit;
}
$db->truncateHostPageDom();
CLI::danger(_('CRAWL_HOST_PAGE_DOM_SELECTORS not provided in the configuration file'));
exit;
CLI::success(_('hostPageDom table successfully truncated'));
exit;
break;
case 'truncate':
break;
default:
$db->truncateHostPageDom();
CLI::danger(_('undefined action argument'));
}
CLI::success(_('hostPageDom table successfully truncated'));
exit;
break;
break;
}
}
break;
}
}
// Default message
@ -446,13 +439,29 @@ CLI::default('/_/\____/\____/\____(_)' ); @@ -446,13 +439,29 @@ CLI::default('/_/\____/\____/\____(_)' );
CLI::break();
CLI::default('available options:');
CLI::break();
CLI::default(' help - this message');
CLI::break();
CLI::default(' db ');
CLI::default(' optimize - optimize all tables');
CLI::break();
CLI::default(' crontab ');
CLI::default(' crawl - execute step in crawler queue');
CLI::default(' clean - execute step in cleaner queue');
CLI::break();
CLI::default(' hostPage ');
CLI::default(' rank - generate hostPage.rank fields');
CLI::break();
CLI::default(' hostPageSnap ');
CLI::default(' repair ');
CLI::default(' db - scan database registry for new or deprecated snap files');
CLI::default(' fs - check all storages for snap files not registered in hostPageSnapStorage, cleanup filesystem');
CLI::default(' reindex - search for host pages without snap records, add found pages to the crawl queue');
CLI::break();
CLI::default(' hostPageDom ');
CLI::default(' generate [selectors] - make hostPageDom index based on related hostPage.data field');
CLI::default(' truncate - flush hostPageDom table');
CLI::default(' help - this message');
CLI::default(' crontab [crawl|clean] - execute crontab script queue');
CLI::default(' hostPage rank reindex - generate rank indexes in hostPage table');
CLI::default(' hostPageSnap repair - sync DB/FS relations');
CLI::default(' hostPageDom generate [selectors] - make hostPageDom index based on related hostPage.data field');
CLI::default(' hostPageDom truncate - flush hostPageDom table');
CLI::break();
CLI::default('get support: https://github.com/YGGverse/YGGo/issues');

Write Preview
Loading…
Cancel
Save