From 1b722812e8690ffcb8aaafd9c7535c23b9291e84 Mon Sep 17 00:00:00 2001 From: ghost Date: Mon, 9 Oct 2023 05:09:18 +0300 Subject: [PATCH] implement torrent scrape queue --- README.md | 2 +- composer.json | 3 +- composer.lock | 73 +++++++++++++++++++------ src/Controller/TorrentController.php | 34 +++++++++--- src/Entity/Torrent.php | 15 ++++++ src/Repository/TorrentRepository.php | 10 ++++ src/Service/TorrentService.php | 81 ++++++++++++++++++++++++++++ 7 files changed, 194 insertions(+), 24 deletions(-) diff --git a/README.md b/README.md index 88a927d..8fae921 100644 --- a/README.md +++ b/README.md @@ -47,7 +47,7 @@ git checkout -b my-pr-branch-name #### Components * [SVG icons](https://icons.getbootstrap.com) -* [Scrapper](https://github.com/medariox/scrapeer) +* [Scrapper](https://github.com/medariox/scrapeer) / [Composer Edition](https://github.com/YGGverse/scrapeer) * [Bencode Library](https://github.com/Rhilip/Bencode) * [Identicons](https://github.com/dmester/jdenticon-php) diff --git a/composer.json b/composer.json index f6fafed..c0d3e3c 100644 --- a/composer.json +++ b/composer.json @@ -47,7 +47,8 @@ "twig/extra-bundle": "^2.12|^3.0", "twig/intl-extra": "^3.7", "twig/string-extra": "^3.7", - "twig/twig": "^2.12|^3.0" + "twig/twig": "^2.12|^3.0", + "yggverse/scrapeer": "^0.5.4" }, "config": { "allow-plugins": { diff --git a/composer.lock b/composer.lock index ea51c06..e3eddf2 100644 --- a/composer.lock +++ b/composer.lock @@ -4,7 +4,7 @@ "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies", "This file is @generated automatically" ], - "content-hash": "d46bb514c4109b10e7327ef9f913c11a", + "content-hash": "3770ffcd80695bc10a22f8ece4f68d1f", "packages": [ { "name": "doctrine/annotations", @@ -177,16 +177,16 @@ }, { "name": "doctrine/collections", - "version": "2.1.3", + "version": "2.1.4", "source": { "type": "git", "url": "https://github.com/doctrine/collections.git", - "reference": "3023e150f90a38843856147b58190aa8b46cc155" + "reference": "72328a11443a0de79967104ad36ba7b30bded134" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/doctrine/collections/zipball/3023e150f90a38843856147b58190aa8b46cc155", - "reference": "3023e150f90a38843856147b58190aa8b46cc155", + "url": "https://api.github.com/repos/doctrine/collections/zipball/72328a11443a0de79967104ad36ba7b30bded134", + "reference": "72328a11443a0de79967104ad36ba7b30bded134", "shasum": "" }, "require": { @@ -194,7 +194,7 @@ "php": "^8.1" }, "require-dev": { - "doctrine/coding-standard": "^10.0", + "doctrine/coding-standard": "^12", "ext-json": "*", "phpstan/phpstan": "^1.8", "phpstan/phpstan-phpunit": "^1.0", @@ -243,7 +243,7 @@ ], "support": { "issues": "https://github.com/doctrine/collections/issues", - "source": "https://github.com/doctrine/collections/tree/2.1.3" + "source": "https://github.com/doctrine/collections/tree/2.1.4" }, "funding": [ { @@ -259,7 +259,7 @@ "type": "tidelift" } ], - "time": "2023-07-06T15:15:36+00:00" + "time": "2023-10-03T09:22:33+00:00" }, { "name": "doctrine/common", @@ -1398,16 +1398,16 @@ }, { "name": "egulias/email-validator", - "version": "4.0.1", + "version": "4.0.2", "source": { "type": "git", "url": "https://github.com/egulias/EmailValidator.git", - "reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff" + "reference": "ebaaf5be6c0286928352e054f2d5125608e5405e" }, "dist": { "type": "zip", - "url": "https://api.github.com/repos/egulias/EmailValidator/zipball/3a85486b709bc384dae8eb78fb2eec649bdb64ff", - "reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff", + "url": "https://api.github.com/repos/egulias/EmailValidator/zipball/ebaaf5be6c0286928352e054f2d5125608e5405e", + "reference": "ebaaf5be6c0286928352e054f2d5125608e5405e", "shasum": "" }, "require": { @@ -1416,8 +1416,8 @@ "symfony/polyfill-intl-idn": "^1.26" }, "require-dev": { - "phpunit/phpunit": "^9.5.27", - "vimeo/psalm": "^4.30" + "phpunit/phpunit": "^10.2", + "vimeo/psalm": "^5.12" }, "suggest": { "ext-intl": "PHP Internationalization Libraries are required to use the SpoofChecking validation" @@ -1453,7 +1453,7 @@ ], "support": { "issues": "https://github.com/egulias/EmailValidator/issues", - "source": "https://github.com/egulias/EmailValidator/tree/4.0.1" + "source": "https://github.com/egulias/EmailValidator/tree/4.0.2" }, "funding": [ { @@ -1461,7 +1461,7 @@ "type": "github" } ], - "time": "2023-01-14T14:17:03+00:00" + "time": "2023-10-06T06:47:41+00:00" }, { "name": "jdenticon/jdenticon", @@ -7604,6 +7604,47 @@ "source": "https://github.com/webmozarts/assert/tree/1.11.0" }, "time": "2022-06-03T18:03:27+00:00" + }, + { + "name": "yggverse/scrapeer", + "version": "0.5.4", + "source": { + "type": "git", + "url": "https://github.com/YGGverse/scrapeer.git", + "reference": "3b9d15c9706a844bf1428812bb938370e907b4f3" + }, + "dist": { + "type": "zip", + "url": "https://api.github.com/repos/YGGverse/scrapeer/zipball/3b9d15c9706a844bf1428812bb938370e907b4f3", + "reference": "3b9d15c9706a844bf1428812bb938370e907b4f3", + "shasum": "" + }, + "type": "library", + "autoload": { + "psr-4": { + "Yggverse\\Scrapeer\\": "src/" + } + }, + "notification-url": "https://packagist.org/downloads/", + "license": [ + "CC-BY-SA-3.0" + ], + "authors": [ + { + "name": "medariox", + "homepage": "https://github.com/medariox/scrapeer" + }, + { + "name": "YGGverse", + "homepage": "https://github.com/YGGverse/scrapeer" + } + ], + "description": "Essential PHP library that scrapes HTTP(S) and UDP trackers for torrent information", + "support": { + "issues": "https://github.com/YGGverse/scrapeer/issues", + "source": "https://github.com/YGGverse/scrapeer/tree/0.5.4" + }, + "time": "2023-10-09T01:47:30+00:00" } ], "packages-dev": [ diff --git a/src/Controller/TorrentController.php b/src/Controller/TorrentController.php index 6a1746f..ea3601a 100644 --- a/src/Controller/TorrentController.php +++ b/src/Controller/TorrentController.php @@ -72,8 +72,8 @@ class TorrentController extends AbstractController 'peers' => (int) $torrent->getPeers(), 'leechers' => (int) $torrent->getLeechers(), ], - 'locales' => $torrentService->findLastTorrentLocales($torrent->getId()), - 'sensitive' => $torrentService->findLastTorrentSensitive($torrent->getId())->isValue(), + 'locales' => $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()), + 'sensitive' => $torrentService->findLastTorrentSensitiveByTorrentId($torrent->getId())->isValue(), 'download' => [ 'file' => @@ -346,7 +346,7 @@ class TorrentController extends AbstractController // Otherwise, get latest available else { - if ($torrentLocales = $torrentService->findLastTorrentLocales($torrent->getId())) + if ($torrentLocales = $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId())) { $torrentLocalesCurrent['userId'] = $torrentLocales->getUserId(); @@ -367,7 +367,7 @@ class TorrentController extends AbstractController // Init edition history $editions = []; - foreach ($torrentService->findTorrentLocales($torrent->getId()) as $torrentLocalesEdition) + foreach ($torrentService->findTorrentLocalesByTorrentId($torrent->getId()) as $torrentLocalesEdition) { $editions[] = [ @@ -652,7 +652,7 @@ class TorrentController extends AbstractController } else { - if ($torrentSensitive = $torrentService->findLastTorrentSensitive($request->get('torrentId'))) + if ($torrentSensitive = $torrentService->findLastTorrentSensitiveByTorrentId($request->get('torrentId'))) { $torrentSensitiveCurrent = [ @@ -675,7 +675,7 @@ class TorrentController extends AbstractController // Init edition history $editions = []; - foreach ($torrentService->findTorrentSensitive($torrent->getId()) as $torrentSensitiveEdition) + foreach ($torrentService->findTorrentSensitiveByTorrentId($torrent->getId()) as $torrentSensitiveEdition) { $editions[] = [ @@ -1088,4 +1088,26 @@ class TorrentController extends AbstractController $file->getMagnetLink() ); } + + // Tools + #[Route( + '/crontab/scrape', + methods: + [ + 'GET' + ] + )] + public function scrape( + Request $request, + TranslatorInterface $translator, + TorrentService $torrentService, + ): Response + { + $torrentService->scrapeTorrentQueue( + explode('|', $this->getParameter('app.trackers')) + ); + + // Render response + return new Response(); // @TODO + } } diff --git a/src/Entity/Torrent.php b/src/Entity/Torrent.php index eac2144..d551b08 100644 --- a/src/Entity/Torrent.php +++ b/src/Entity/Torrent.php @@ -20,6 +20,9 @@ class Torrent #[ORM\Column] private ?int $added = null; + #[ORM\Column(nullable: true)] + private ?int $scraped = null; + #[ORM\Column] private ?bool $approved = null; @@ -71,6 +74,18 @@ class Torrent return $this; } + public function getScraped(): ?int + { + return $this->scraped; + } + + public function setScraped(int $scraped): static + { + $this->scraped = $scraped; + + return $this; + } + public function getKeywords(): ?string { return $this->keywords; diff --git a/src/Repository/TorrentRepository.php b/src/Repository/TorrentRepository.php index 2ab05f6..d40022b 100644 --- a/src/Repository/TorrentRepository.php +++ b/src/Repository/TorrentRepository.php @@ -30,4 +30,14 @@ class TorrentRepository extends ServiceEntityRepository ->getOneOrNullResult() ; } + + public function getTorrentScrapeQueue(): ?Torrent + { + return $this->createQueryBuilder('t') + ->orderBy('t.scraped', 'ASC') // same to ts.added + ->setMaxResults(1) + ->getQuery() + ->getOneOrNullResult() + ; + } } diff --git a/src/Service/TorrentService.php b/src/Service/TorrentService.php index 8abf189..d1231bf 100644 --- a/src/Service/TorrentService.php +++ b/src/Service/TorrentService.php @@ -36,6 +36,80 @@ class TorrentService } // Tools + public function scrapeTorrentQueue( + array $trackers = [] + ): void + { + // Init Scraper + $scraper = new \Yggverse\Scrapeer\Scraper(); + + if ($torrent = $this->getTorrentScrapeQueue()) + { + // Get file + if (!$file = $this->readTorrentFileByTorrentId($torrent->getId())) + { + // @TODO + throw new \Exception( + $translator->trans('File not found') + ); + } + + // Get info hashes + $hashes = []; + + if ($hash = $file->getInfoHashV1(false)) + { + $hashes[] = $hash; + } + + if ($hash = $file->getInfoHashV2(false)) + { + $hashes[] = $hash; + } + + // Get scrape + if ($hashes && $trackers) + { + // Update scrape info + if ($results = $scraper->scrape($hashes, $trackers, null, 1)) + { + foreach ($results as $result) + { + if (isset($result['seeders'])) + { + $torrent->setSeeders( + (int) $result['seeders'] + ); + } + + if (isset($result['completed'])) + { + $torrent->setPeers( + (int) $result['completed'] + ); + } + + if (isset($result['leechers'])) + { + $torrent->setLeechers( + (int) $result['leechers'] + ); + } + } + } + } + + // Update time scraped + $torrent->setScraped( + time() + ); + + // Save results to DB + $this->entityManagerInterface->persist($torrent); + $this->entityManagerInterface->flush(); + } + } + public function readTorrentFileByFilepath( string $filepath ): ?\Rhilip\Bencode\TorrentFile @@ -188,6 +262,13 @@ class TorrentService return $torrent; } + public function getTorrentScrapeQueue(): ?Torrent + { + return $this->entityManagerInterface + ->getRepository(Torrent::class) + ->getTorrentScrapeQueue(); + } + // Torrent locale public function getTorrentLocales(int $id): ?TorrentLocales {