Browse Source

improve debug

main
ghost 1 year ago
parent
commit
4dcdafc638
  1. 632
      src/crontab/import/feed.php

632
src/crontab/import/feed.php

@ -5,7 +5,7 @@ $semaphore = sem_get(crc32('yggtracker.crontab.import.feed'), 1);
if (false === sem_acquire($semaphore, true)) if (false === sem_acquire($semaphore, true))
{ {
exit (PHP_EOL . 'yggtracker.crontab.import.feed process locked by another thread.' . PHP_EOL); exit(_('yggtracker.crontab.import.feed process locked by another thread.'));
} }
// Bootstrap // Bootstrap
@ -13,19 +13,40 @@ require_once __DIR__ . '/../../config/bootstrap.php';
if (empty(API_IMPORT_ENABLED)) if (empty(API_IMPORT_ENABLED))
{ {
exit; exit(_('Import disabled in settings'));
} }
// Init Debug // Init debug
$debug = $debug =
[ [
'time' => [ 'data' => [],
'time' =>
[
'ISO8601' => date('c'), 'ISO8601' => date('c'),
'total' => microtime(true), 'total' => microtime(true),
], ],
'http' =>
[
'total' => 0,
],
'memory' =>
[
'start' => memory_get_usage(),
'total' => 0,
'peaks' => 0
],
'db' =>
[
'total' => [
'select' => 0,
'insert' => 0,
'update' => 0,
'delete' => 0,
]
],
]; ];
// Begin export // Begin import
try try
{ {
// Transaction begin // Transaction begin
@ -36,8 +57,22 @@ try
) as $node) ) as $node)
{ {
// Skip non-condition addresses // Skip non-condition addresses
if (!Valid::url($node->manifest)) $error = [];
if (!Valid::url($node->manifest, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Manifest URL "%s" invalid: %s'),
$node->manifest,
print_r(
$error,
true
)
)
);
continue; continue;
} }
@ -49,45 +84,107 @@ try
empty($manifestUrl->host->name) || empty($manifestUrl->host->name) ||
$manifestUrl->host->name == $thisUrl->host->name) // @TODO some mirrors could be available, improve condition $manifestUrl->host->name == $thisUrl->host->name) // @TODO some mirrors could be available, improve condition
{ {
// No debug warnings in this case, continue next item
continue; continue;
} }
// Get node manifest // Get node manifest
$curl = new Curl($node->manifest, API_USER_AGENT); $curl = new Curl($node->manifest, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Manifest URL "%s" unreachable with code: "%s"'),
$node->manifest,
$code
)
);
continue; continue;
} }
if (!$manifest = $curl->getResponse()) if (!$manifest = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Manifest URL "%s" has broken response'),
$node->manifest
)
);
continue; continue;
} }
if (empty($manifest->export)) if (empty($manifest->export))
{ {
array_push(
$debug['data'],
sprintf(
_('Manifest URL "%s" has broken protocol'),
$node->manifest
)
);
continue; continue;
} }
// Users // Users
if (API_IMPORT_USERS_ENABLED) if (API_IMPORT_USERS_ENABLED)
{ {
if (Valid::url($manifest->export->users)) $error = [];
if (!Valid::url($manifest->export->users, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Users feed URL "%s" invalid: %s'),
$manifest->export->users,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->users, API_USER_AGENT); $curl = new Curl($manifest->export->users, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Users feed URL "%s" unreachable with code: "%s"'),
$manifest->export->users,
$code
)
);
continue; continue;
} }
if (!$remoteUsers = $curl->getResponse()) if (!$remoteUsers = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Users feed URL "%s" has broken response'),
$manifest->export->users
)
);
continue; continue;
} }
@ -97,23 +194,55 @@ try
foreach ((object) $remoteUsers as $remoteUser) foreach ((object) $remoteUsers as $remoteUser)
{ {
// Validate required fields // Validate required fields
if (!Valid::user($remoteUser)) $error = [];
if (!Valid::user($remoteUser, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Users feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->users,
print_r(
$remoteUser,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Skip import on user approved required // Skip import on user approved required
if (API_IMPORT_USERS_APPROVED_ONLY && !$remoteUser->approved) if (API_IMPORT_USERS_APPROVED_ONLY && !$remoteUser->approved)
{ {
// No debug warnings in this case, continue next item
continue; continue;
} }
// Init session // Init session
else if (!$localUser = $db->getUser( else if (!$localUser = $db->getUser(
$db->initUserId($remoteUser->address, $db->initUserId(
$remoteUser->address,
USER_AUTO_APPROVE_ON_IMPORT_APPROVED ? $remoteUser->approved : USER_DEFAULT_APPROVED, USER_AUTO_APPROVE_ON_IMPORT_APPROVED ? $remoteUser->approved : USER_DEFAULT_APPROVED,
$remoteUser->timeAdded))) $remoteUser->timeAdded
{ )
))
{
array_push(
$debug['data'],
sprintf(
_('Could not init user with address "%s" using feed URL "%s"'),
$remoteUser->address,
$manifest->export->users
)
);
continue; continue;
} }
@ -163,21 +292,54 @@ try
// Magnets // Magnets
if (API_IMPORT_MAGNETS_ENABLED) if (API_IMPORT_MAGNETS_ENABLED)
{ {
if (Valid::url($manifest->export->magnets)) $error = [];
if (!Valid::url($manifest->export->magnets, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnets feed URL "%s" invalid: %s'),
$manifest->export->magnets,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->magnets, API_USER_AGENT); $curl = new Curl($manifest->export->magnets, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnets feed URL "%s" unreachable with code: "%s"'),
$manifest->export->magnets,
$code
)
);
continue; continue;
} }
if (!$remoteMagnets = $curl->getResponse()) if (!$remoteMagnets = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnets feed URL "%s" has broken response'),
$manifest->export->magnets
)
);
continue; continue;
} }
@ -187,20 +349,53 @@ try
foreach ((object) $remoteMagnets as $remoteMagnet) foreach ((object) $remoteMagnets as $remoteMagnet)
{ {
// Validate required fields by protocol // Validate required fields by protocol
if (!Valid::magnet($remoteMagnet)) $error = [];
if (!Valid::magnet($remoteMagnet, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnets feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->magnets,
print_r(
$remoteMagnet,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Aliases check // Aliases check
if (!isset($aliasUserId[$remoteMagnet->userId])) if (!isset($aliasUserId[$remoteMagnet->userId]))
{ {
array_push(
$debug['data'],
sprintf(
_('Local alias for remote userId "%s" not found in URL "%s" %s'),
$remoteMagnet->userId,
$manifest->export->magnets,
print_r(
$remoteMagnet,
true
),
)
);
continue; continue;
} }
// Skip import on magnet approved required // Skip import on magnet approved required
if (API_IMPORT_MAGNETS_APPROVED_ONLY && !$remoteMagnet->approved) if (API_IMPORT_MAGNETS_APPROVED_ONLY && !$remoteMagnet->approved)
{ {
// No debug warnings in this case, continue next item
continue; continue;
} }
@ -255,8 +450,6 @@ try
{ {
case 1: case 1:
if (Yggverse\Parser\Magnet::isXTv1($xt->value))
{
$exist = false; $exist = false;
foreach ($db->findMagnetToInfoHashByMagnetId($localMagnet->magnetId) as $result) foreach ($db->findMagnetToInfoHashByMagnetId($localMagnet->magnetId) as $result)
@ -275,18 +468,15 @@ try
$db->addMagnetToInfoHash( $db->addMagnetToInfoHash(
$localMagnet->magnetId, $localMagnet->magnetId,
$db->initInfoHashId( $db->initInfoHashId(
Yggverse\Parser\Magnet::filterInfoHash($xt->value), 1 $xt->value, 1
) )
); );
} }
}
break; break;
case 2: case 2:
if (Yggverse\Parser\Magnet::isXTv2($xt->value))
{
$exist = false; $exist = false;
foreach ($db->findMagnetToInfoHashByMagnetId($localMagnet->magnetId) as $result) foreach ($db->findMagnetToInfoHashByMagnetId($localMagnet->magnetId) as $result)
@ -305,11 +495,10 @@ try
$db->addMagnetToInfoHash( $db->addMagnetToInfoHash(
$localMagnet->magnetId, $localMagnet->magnetId,
$db->initInfoHashId( $db->initInfoHashId(
Yggverse\Parser\Magnet::filterInfoHash($xt->value), 2 $xt->value, 2
) )
); );
} }
}
break; break;
} }
@ -320,12 +509,14 @@ try
{ {
$db->initMagnetToKeywordTopicId( $db->initMagnetToKeywordTopicId(
$localMagnet->magnetId, $localMagnet->magnetId,
$db->initKeywordTopicId(trim(mb_strtolower(strip_tags(html_entity_decode($kt))))) $db->initKeywordTopicId(trim(mb_strtolower($kt))) // @TODO
); );
} }
// tr // tr
foreach ($remoteMagnet->tr as $tr) foreach ($remoteMagnet->tr as $tr)
{
if ($url = Yggverse\Parser\Url::parse($tr))
{ {
$db->initMagnetToAddressTrackerId( $db->initMagnetToAddressTrackerId(
$localMagnet->magnetId, $localMagnet->magnetId,
@ -337,9 +528,12 @@ try
) )
); );
} }
}
// as // as
foreach ($remoteMagnet->as as $as) foreach ($remoteMagnet->as as $as)
{
if ($url = Yggverse\Parser\Url::parse($as))
{ {
$db->initMagnetToAcceptableSourceId( $db->initMagnetToAcceptableSourceId(
$localMagnet->magnetId, $localMagnet->magnetId,
@ -351,9 +545,12 @@ try
) )
); );
} }
}
// xs // xs
foreach ($remoteMagnet->xs as $xs) foreach ($remoteMagnet->xs as $xs)
{
if ($url = Yggverse\Parser\Url::parse($tr))
{ {
$db->initMagnetToExactSourceId( $db->initMagnetToExactSourceId(
$localMagnet->magnetId, $localMagnet->magnetId,
@ -366,6 +563,7 @@ try
); );
} }
} }
}
// Add magnet alias for this host // Add magnet alias for this host
$aliasMagnetId[$remoteMagnet->magnetId] = $localMagnet->magnetId; $aliasMagnetId[$remoteMagnet->magnetId] = $localMagnet->magnetId;
@ -374,41 +572,125 @@ try
// Magnet comments // Magnet comments
if (API_IMPORT_MAGNET_COMMENTS_ENABLED) if (API_IMPORT_MAGNET_COMMENTS_ENABLED)
{ {
if (Valid::url($manifest->export->magnetComments)) $error = [];
if (!Valid::url($manifest->export->magnetComments, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet comments feed URL "%s" invalid: %s'),
$manifest->export->magnetComments,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->magnetComments, API_USER_AGENT); $curl = new Curl($manifest->export->magnetComments, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet comments feed URL "%s" unreachable with code: "%s"'),
$manifest->export->magnetComments,
$code
)
);
continue; continue;
} }
if (!$remoteMagnetComments = $curl->getResponse()) if (!$remoteMagnetComments = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet comments feed URL "%s" has broken response'),
$manifest->export->magnetComments
)
);
continue; continue;
} }
foreach ((object) $remoteMagnetComments as $remoteMagnetComment) foreach ((object) $remoteMagnetComments as $remoteMagnetComment)
{ {
// Validate // Validate
if (!Valid::magnetComment($remoteMagnetComment)) $error = [];
if (!Valid::magnetComment($remoteMagnetComment, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet comments feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->magnetComments,
print_r(
$remoteMagnetComment,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Aliases check // Aliases check
if (!isset($aliasMagnetId[$remoteMagnetComment->magnetId]) || !isset($aliasUserId[$remoteMagnetComment->userId])) if (!isset($aliasUserId[$remoteMagnetComment->userId]))
{
array_push(
$debug['data'],
sprintf(
_('Local alias for remote userId "%s" not found in URL "%s" %s'),
$remoteMagnetComment->userId,
$manifest->export->magnetComments,
print_r(
$remoteMagnetComment,
true
),
)
);
continue;
}
if (!isset($aliasMagnetId[$remoteMagnetComment->magnetId]))
{ {
array_push(
$debug['data'],
sprintf(
_('Local alias for remote magnetId "%s" not found in URL "%s" %s'),
$remoteMagnetComment->magnetId,
$manifest->export->magnetComments,
print_r(
$remoteMagnetComment,
true
),
)
);
continue; continue;
} }
// Skip import on magnet comment approved required // Skip import on magnet comment approved required
if (API_IMPORT_MAGNET_COMMENTS_APPROVED_ONLY && !$remoteMagnetComment->approved) if (API_IMPORT_MAGNET_COMMENTS_APPROVED_ONLY && !$remoteMagnetComment->approved)
{ {
// No debug warnings in this case, continue next item
continue; continue;
} }
@ -444,35 +726,118 @@ try
// Magnet downloads // Magnet downloads
if (API_IMPORT_MAGNET_DOWNLOADS_ENABLED) if (API_IMPORT_MAGNET_DOWNLOADS_ENABLED)
{ {
if (Valid::url($manifest->export->magnetDownloads)) // Skip non-condition addresses
$error = [];
if (!Valid::url($manifest->export->magnetDownloads, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet downloads feed URL "%s" invalid: %s'),
$manifest->export->magnetDownloads,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->magnetDownloads, API_USER_AGENT); $curl = new Curl($manifest->export->magnetDownloads, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet downloads feed URL "%s" unreachable with code: "%s"'),
$manifest->export->magnetDownloads,
$code
)
);
continue; continue;
} }
if (!$remoteMagnetDownloads = $curl->getResponse()) if (!$remoteMagnetDownloads = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet downloads feed URL "%s" has broken response'),
$manifest->export->magnetDownloads
)
);
continue; continue;
} }
foreach ((object) $remoteMagnetDownloads as $remoteMagnetDownload) foreach ((object) $remoteMagnetDownloads as $remoteMagnetDownload)
{ {
// Validate // Validate
if (!Valid::magnetDownload($remoteMagnetDownload)) $error = [];
if (!Valid::magnetDownload($remoteMagnetDownload, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet downloads feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->magnetDownloads,
print_r(
$remoteMagnetDownload,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Aliases check // Aliases check
if (!isset($aliasMagnetId[$remoteMagnetDownload->magnetId]) || !isset($aliasUserId[$remoteMagnetDownload->userId])) if (!isset($aliasUserId[$remoteMagnetDownload->userId]))
{
array_push(
$debug['data'],
sprintf(
_('Local alias for remote userId "%s" not found in URL "%s" %s'),
$remoteMagnetDownload->userId,
$manifest->export->magnetDownloads,
print_r(
$remoteMagnetDownload,
true
),
)
);
continue;
}
if (!isset($aliasMagnetId[$remoteMagnetDownload->magnetId]))
{ {
array_push(
$debug['data'],
sprintf(
_('Local alias for remote magnetId "%s" not found in URL "%s" %s'),
$remoteMagnetDownload->magnetId,
$manifest->export->magnetDownloads,
print_r(
$remoteMagnetDownload,
true
),
)
);
continue; continue;
} }
@ -493,35 +858,117 @@ try
// Magnet views // Magnet views
if (API_IMPORT_MAGNET_VIEWS_ENABLED) if (API_IMPORT_MAGNET_VIEWS_ENABLED)
{ {
if (Valid::url($manifest->export->magnetViews)) $error = [];
if (!Valid::url($manifest->export->magnetViews, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet views feed URL "%s" invalid: %s'),
$manifest->export->magnetViews,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->magnetViews, API_USER_AGENT); $curl = new Curl($manifest->export->magnetViews, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet views feed URL "%s" unreachable with code: "%s"'),
$manifest->export->magnetViews,
$code
)
);
continue; continue;
} }
if (!$remoteMagnetViews = $curl->getResponse()) if (!$remoteMagnetViews = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet views feed URL "%s" has broken response'),
$manifest->export->magnetViews
)
);
continue; continue;
} }
foreach ((object) $remoteMagnetViews as $remoteMagnetView) foreach ((object) $remoteMagnetViews as $remoteMagnetView)
{ {
// Validate // Validate
if (!Valid::magnetView($remoteMagnetView)) $error = [];
if (!Valid::magnetView($remoteMagnetView, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet views feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->magnetViews,
print_r(
$remoteMagnetView,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Aliases check // Aliases check
if (!isset($aliasMagnetId[$remoteMagnetView->magnetId]) || !isset($aliasUserId[$remoteMagnetView->userId])) if (!isset($aliasUserId[$remoteMagnetView->userId]))
{
array_push(
$debug['data'],
sprintf(
_('Local alias for remote userId "%s" not found in URL "%s" %s'),
$remoteMagnetView->userId,
$manifest->export->magnetViews,
print_r(
$remoteMagnetView,
true
),
)
);
continue;
}
if (!isset($aliasMagnetId[$remoteMagnetView->magnetId]))
{ {
array_push(
$debug['data'],
sprintf(
_('Local alias for remote magnetId "%s" not found in URL "%s" %s'),
$remoteMagnetView->magnetId,
$manifest->export->magnetViews,
print_r(
$remoteMagnetView,
true
),
)
);
continue; continue;
} }
@ -542,35 +989,117 @@ try
// Magnet stars // Magnet stars
if (API_IMPORT_MAGNET_STARS_ENABLED) if (API_IMPORT_MAGNET_STARS_ENABLED)
{ {
if (Valid::url($manifest->export->magnetStars)) $error = [];
if (!Valid::url($manifest->export->magnetStars, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet stars feed URL "%s" invalid: %s'),
$manifest->export->magnetStars,
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Call feed // Call feed
$curl = new Curl($manifest->export->magnetStars, API_USER_AGENT); $curl = new Curl($manifest->export->magnetStars, API_USER_AGENT);
if (200 != $curl->getCode()) $debug['http']['total']++;
if (200 != $code = $curl->getCode())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet stars feed URL "%s" unreachable with code: "%s"'),
$manifest->export->magnetStars,
$code
)
);
continue; continue;
} }
if (!$remoteMagnetStars = $curl->getResponse()) if (!$remoteMagnetStars = $curl->getResponse())
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet stars feed URL "%s" has broken response'),
$manifest->export->magnetStars
)
);
continue; continue;
} }
foreach ((object) $remoteMagnetStars as $remoteMagnetStar) foreach ((object) $remoteMagnetStars as $remoteMagnetStar)
{ {
// Validate // Validate
if (!Valid::magnetStar($remoteMagnetStar)) $error = [];
if (!Valid::magnetStar($remoteMagnetStar, $error))
{ {
array_push(
$debug['data'],
sprintf(
_('Magnet stars feed URL "%s" has invalid protocol: "%s" error: "%s"'),
$manifest->export->magnetStars,
print_r(
$remoteMagnetStar,
true
),
print_r(
$error,
true
)
)
);
continue; continue;
} }
// Aliases check // Aliases check
if (!isset($aliasMagnetId[$remoteMagnetStar->magnetId]) || !isset($aliasUserId[$remoteMagnetStar->userId])) if (!isset($aliasUserId[$remoteMagnetStar->userId]))
{
array_push(
$debug['data'],
sprintf(
_('Local alias for remote userId "%s" not found in URL "%s" %s'),
$remoteMagnetStar->userId,
$manifest->export->magnetStars,
print_r(
$remoteMagnetStar,
true
),
)
);
continue;
}
if (!isset($aliasMagnetId[$remoteMagnetStar->magnetId]))
{ {
array_push(
$debug['data'],
sprintf(
_('Local alias for remote magnetId "%s" not found in URL "%s" %s'),
$remoteMagnetStar->magnetId,
$manifest->export->magnetStars,
print_r(
$remoteMagnetStar,
true
),
)
);
continue; continue;
} }
@ -604,15 +1133,12 @@ try
// Debug output // Debug output
$debug['time']['total'] = microtime(true) - $debug['time']['total']; $debug['time']['total'] = microtime(true) - $debug['time']['total'];
print_r( $debug['memory']['total'] = memory_get_usage() - $debug['memory']['start'];
array_merge($debug, [ $debug['memory']['peaks'] = memory_get_peak_usage();
'db' => [
'total' => [ $debug['db']['total']['select'] = $db->getDebug()->query->select->total;
'select' => $db->getDebug()->query->select->total, $debug['db']['total']['insert'] = $db->getDebug()->query->insert->total;
'insert' => $db->getDebug()->query->insert->total, $debug['db']['total']['update'] = $db->getDebug()->query->update->total;
'update' => $db->getDebug()->query->update->total, $debug['db']['total']['delete'] = $db->getDebug()->query->delete->total;
'delete' => $db->getDebug()->query->delete->total,
] print_r($debug);
]
])
);
Loading…
Cancel
Save