YGGverse/YGGtracker
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGtracker.git synced 2025-02-03 10:35:32 +00:00
Code Issues Projects Releases Wiki Activity

implement torrent scrape queue

Browse Source
This commit is contained in:
ghost 2023-10-09 05:09:18 +03:00
parent d2f7fff24e
commit 1b722812e8
7 changed files with 194 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -47,7 +47,7 @@ git checkout -b my-pr-branch-name
#### Components
* [SVG icons](https://icons.getbootstrap.com)
* [Scrapper](https://github.com/medariox/scrapeer)
* [Scrapper](https://github.com/medariox/scrapeer) / [Composer Edition](https://github.com/YGGverse/scrapeer)
* [Bencode Library](https://github.com/Rhilip/Bencode)
* [Identicons](https://github.com/dmester/jdenticon-php)

3
composer.json
View File

@ -47,7 +47,8 @@
"twig/extra-bundle": "^2.12|^3.0",
"twig/intl-extra": "^3.7",
"twig/string-extra": "^3.7",
"twig/twig": "^2.12|^3.0"
"twig/twig": "^2.12|^3.0",
"yggverse/scrapeer": "^0.5.4"
},
"config": {
"allow-plugins": {

73
composer.lock generated
View File

@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "d46bb514c4109b10e7327ef9f913c11a",
"content-hash": "3770ffcd80695bc10a22f8ece4f68d1f",
"packages": [
{
"name": "doctrine/annotations",
@ -177,16 +177,16 @@
},
{
"name": "doctrine/collections",
"version": "2.1.3",
"version": "2.1.4",
"source": {
"type": "git",
"url": "https://github.com/doctrine/collections.git",
"reference": "3023e150f90a38843856147b58190aa8b46cc155"
"reference": "72328a11443a0de79967104ad36ba7b30bded134"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/collections/zipball/3023e150f90a38843856147b58190aa8b46cc155",
"reference": "3023e150f90a38843856147b58190aa8b46cc155",
"url": "https://api.github.com/repos/doctrine/collections/zipball/72328a11443a0de79967104ad36ba7b30bded134",
"reference": "72328a11443a0de79967104ad36ba7b30bded134",
"shasum": ""
},
"require": {
@ -194,7 +194,7 @@
"php": "^8.1"
},
"require-dev": {
"doctrine/coding-standard": "^10.0",
"doctrine/coding-standard": "^12",
"ext-json": "*",
"phpstan/phpstan": "^1.8",
"phpstan/phpstan-phpunit": "^1.0",
@ -243,7 +243,7 @@
],
"support": {
"issues": "https://github.com/doctrine/collections/issues",
"source": "https://github.com/doctrine/collections/tree/2.1.3"
"source": "https://github.com/doctrine/collections/tree/2.1.4"
},
"funding": [
{
@ -259,7 +259,7 @@
"type": "tidelift"
}
],
"time": "2023-07-06T15:15:36+00:00"
"time": "2023-10-03T09:22:33+00:00"
},
{
"name": "doctrine/common",
@ -1398,16 +1398,16 @@
},
{
"name": "egulias/email-validator",
"version": "4.0.1",
"version": "4.0.2",
"source": {
"type": "git",
"url": "https://github.com/egulias/EmailValidator.git",
"reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff"
"reference": "ebaaf5be6c0286928352e054f2d5125608e5405e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/egulias/EmailValidator/zipball/3a85486b709bc384dae8eb78fb2eec649bdb64ff",
"reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff",
"url": "https://api.github.com/repos/egulias/EmailValidator/zipball/ebaaf5be6c0286928352e054f2d5125608e5405e",
"reference": "ebaaf5be6c0286928352e054f2d5125608e5405e",
"shasum": ""
},
"require": {
@ -1416,8 +1416,8 @@
"symfony/polyfill-intl-idn": "^1.26"
},
"require-dev": {
"phpunit/phpunit": "^9.5.27",
"vimeo/psalm": "^4.30"
"phpunit/phpunit": "^10.2",
"vimeo/psalm": "^5.12"
},
"suggest": {
"ext-intl": "PHP Internationalization Libraries are required to use the SpoofChecking validation"
@ -1453,7 +1453,7 @@
],
"support": {
"issues": "https://github.com/egulias/EmailValidator/issues",
"source": "https://github.com/egulias/EmailValidator/tree/4.0.1"
"source": "https://github.com/egulias/EmailValidator/tree/4.0.2"
},
"funding": [
{
@ -1461,7 +1461,7 @@
"type": "github"
}
],
"time": "2023-01-14T14:17:03+00:00"
"time": "2023-10-06T06:47:41+00:00"
},
{
"name": "jdenticon/jdenticon",
@ -7604,6 +7604,47 @@
"source": "https://github.com/webmozarts/assert/tree/1.11.0"
},
"time": "2022-06-03T18:03:27+00:00"
},
{
"name": "yggverse/scrapeer",
"version": "0.5.4",
"source": {
"type": "git",
"url": "https://github.com/YGGverse/scrapeer.git",
"reference": "3b9d15c9706a844bf1428812bb938370e907b4f3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/YGGverse/scrapeer/zipball/3b9d15c9706a844bf1428812bb938370e907b4f3",
"reference": "3b9d15c9706a844bf1428812bb938370e907b4f3",
"shasum": ""
},
"type": "library",
"autoload": {
"psr-4": {
"Yggverse\\Scrapeer\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"CC-BY-SA-3.0"
],
"authors": [
{
"name": "medariox",
"homepage": "https://github.com/medariox/scrapeer"
},
{
"name": "YGGverse",
"homepage": "https://github.com/YGGverse/scrapeer"
}
],
"description": "Essential PHP library that scrapes HTTP(S) and UDP trackers for torrent information",
"support": {
"issues": "https://github.com/YGGverse/scrapeer/issues",
"source": "https://github.com/YGGverse/scrapeer/tree/0.5.4"
},
"time": "2023-10-09T01:47:30+00:00"
}
],
"packages-dev": [

34
src/Controller/TorrentController.php
View File

@ -72,8 +72,8 @@ class TorrentController extends AbstractController
'peers' => (int) $torrent->getPeers(),
'leechers' => (int) $torrent->getLeechers(),
],
'locales' => $torrentService->findLastTorrentLocales($torrent->getId()),
'sensitive' => $torrentService->findLastTorrentSensitive($torrent->getId())->isValue(),
'locales' => $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()),
'sensitive' => $torrentService->findLastTorrentSensitiveByTorrentId($torrent->getId())->isValue(),
'download' =>
[
'file' =>
@ -346,7 +346,7 @@ class TorrentController extends AbstractController
// Otherwise, get latest available
else
{
if ($torrentLocales = $torrentService->findLastTorrentLocales($torrent->getId()))
if ($torrentLocales = $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()))
{
$torrentLocalesCurrent['userId'] = $torrentLocales->getUserId();
@ -367,7 +367,7 @@ class TorrentController extends AbstractController
// Init edition history
$editions = [];
foreach ($torrentService->findTorrentLocales($torrent->getId()) as $torrentLocalesEdition)
foreach ($torrentService->findTorrentLocalesByTorrentId($torrent->getId()) as $torrentLocalesEdition)
{
$editions[] =
[
@ -652,7 +652,7 @@ class TorrentController extends AbstractController
}
else
{
if ($torrentSensitive = $torrentService->findLastTorrentSensitive($request->get('torrentId')))
if ($torrentSensitive = $torrentService->findLastTorrentSensitiveByTorrentId($request->get('torrentId')))
{
$torrentSensitiveCurrent =
[
@ -675,7 +675,7 @@ class TorrentController extends AbstractController
// Init edition history
$editions = [];
foreach ($torrentService->findTorrentSensitive($torrent->getId()) as $torrentSensitiveEdition)
foreach ($torrentService->findTorrentSensitiveByTorrentId($torrent->getId()) as $torrentSensitiveEdition)
{
$editions[] =
[
@ -1088,4 +1088,26 @@ class TorrentController extends AbstractController
$file->getMagnetLink()
);
}
// Tools
#[Route(
'/crontab/scrape',
methods:
[
'GET'
]
)]
public function scrape(
Request $request,
TranslatorInterface $translator,
TorrentService $torrentService,
): Response
{
$torrentService->scrapeTorrentQueue(
explode('|', $this->getParameter('app.trackers'))
);
// Render response
return new Response(); // @TODO
}
}

15
src/Entity/Torrent.php
View File

@ -20,6 +20,9 @@ class Torrent
#[ORM\Column]
private ?int $added = null;
#[ORM\Column(nullable: true)]
private ?int $scraped = null;
#[ORM\Column]
private ?bool $approved = null;
@ -71,6 +74,18 @@ class Torrent
return $this;
}
public function getScraped(): ?int
{
return $this->scraped;
}
public function setScraped(int $scraped): static
{
$this->scraped = $scraped;
return $this;
}
public function getKeywords(): ?string
{
return $this->keywords;

10
src/Repository/TorrentRepository.php
View File

@ -30,4 +30,14 @@ class TorrentRepository extends ServiceEntityRepository
->getOneOrNullResult()
;
}
public function getTorrentScrapeQueue(): ?Torrent
{
return $this->createQueryBuilder('t')
->orderBy('t.scraped', 'ASC') // same to ts.added
->setMaxResults(1)
->getQuery()
->getOneOrNullResult()
;
}
}

81
src/Service/TorrentService.php
View File

@ -36,6 +36,80 @@ class TorrentService
}
// Tools
public function scrapeTorrentQueue(
array $trackers = []
): void
{
// Init Scraper
$scraper = new \Yggverse\Scrapeer\Scraper();
if ($torrent = $this->getTorrentScrapeQueue())
{
// Get file
if (!$file = $this->readTorrentFileByTorrentId($torrent->getId()))
{
// @TODO
throw new \Exception(
$translator->trans('File not found')
);
}
// Get info hashes
$hashes = [];
if ($hash = $file->getInfoHashV1(false))
{
$hashes[] = $hash;
}
if ($hash = $file->getInfoHashV2(false))
{
$hashes[] = $hash;
}
// Get scrape
if ($hashes && $trackers)
{
// Update scrape info
if ($results = $scraper->scrape($hashes, $trackers, null, 1))
{
foreach ($results as $result)
{
if (isset($result['seeders']))
{
$torrent->setSeeders(
(int) $result['seeders']
);
}
if (isset($result['completed']))
{
$torrent->setPeers(
(int) $result['completed']
);
}
if (isset($result['leechers']))
{
$torrent->setLeechers(
(int) $result['leechers']
);
}
}
}
}
// Update time scraped
$torrent->setScraped(
time()
);
// Save results to DB
$this->entityManagerInterface->persist($torrent);
$this->entityManagerInterface->flush();
}
}
public function readTorrentFileByFilepath(
string $filepath
): ?\Rhilip\Bencode\TorrentFile
@ -188,6 +262,13 @@ class TorrentService
return $torrent;
}
public function getTorrentScrapeQueue(): ?Torrent
{
return $this->entityManagerInterface
->getRepository(Torrent::class)
->getTorrentScrapeQueue();
}
// Torrent locale
public function getTorrentLocales(int $id): ?TorrentLocales
{