implement torrent scrape queue

README.md

@@ -47,7 +47,7 @@ git checkout -b my-pr-branch-name
 #### Components
 
 * [SVG icons](https://icons.getbootstrap.com)
-* [Scrapper](https://github.com/medariox/scrapeer)
+* [Scrapper](https://github.com/medariox/scrapeer) / [Composer Edition](https://github.com/YGGverse/scrapeer)
 * [Bencode Library](https://github.com/Rhilip/Bencode)
 * [Identicons](https://github.com/dmester/jdenticon-php)
 

composer.json

@@ -47,7 +47,8 @@
         "twig/extra-bundle": "^2.12|^3.0",
         "twig/intl-extra": "^3.7",
         "twig/string-extra": "^3.7",
-        "twig/twig": "^2.12|^3.0"
+        "twig/twig": "^2.12|^3.0",
+        "yggverse/scrapeer": "^0.5.4"
     },
     "config": {
         "allow-plugins": {

composer.lock

@@ -4,7 +4,7 @@
         "Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
         "This file is @generated automatically"
     ],
-    "content-hash": "d46bb514c4109b10e7327ef9f913c11a",
+    "content-hash": "3770ffcd80695bc10a22f8ece4f68d1f",
     "packages": [
         {
             "name": "doctrine/annotations",
@@ -177,16 +177,16 @@
         },
         {
             "name": "doctrine/collections",
-            "version": "2.1.3",
+            "version": "2.1.4",
             "source": {
                 "type": "git",
                 "url": "https://github.com/doctrine/collections.git",
-                "reference": "3023e150f90a38843856147b58190aa8b46cc155"
+                "reference": "72328a11443a0de79967104ad36ba7b30bded134"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/doctrine/collections/zipball/3023e150f90a38843856147b58190aa8b46cc155",
+                "url": "https://api.github.com/repos/doctrine/collections/zipball/72328a11443a0de79967104ad36ba7b30bded134",
-                "reference": "3023e150f90a38843856147b58190aa8b46cc155",
+                "reference": "72328a11443a0de79967104ad36ba7b30bded134",
                 "shasum": ""
             },
             "require": {
@@ -194,7 +194,7 @@
                 "php": "^8.1"
             },
             "require-dev": {
-                "doctrine/coding-standard": "^10.0",
+                "doctrine/coding-standard": "^12",
                 "ext-json": "*",
                 "phpstan/phpstan": "^1.8",
                 "phpstan/phpstan-phpunit": "^1.0",
@@ -243,7 +243,7 @@
             ],
             "support": {
                 "issues": "https://github.com/doctrine/collections/issues",
-                "source": "https://github.com/doctrine/collections/tree/2.1.3"
+                "source": "https://github.com/doctrine/collections/tree/2.1.4"
             },
             "funding": [
                 {
@@ -259,7 +259,7 @@
                     "type": "tidelift"
                 }
             ],
-            "time": "2023-07-06T15:15:36+00:00"
+            "time": "2023-10-03T09:22:33+00:00"
         },
         {
             "name": "doctrine/common",
@@ -1398,16 +1398,16 @@
         },
         {
             "name": "egulias/email-validator",
-            "version": "4.0.1",
+            "version": "4.0.2",
             "source": {
                 "type": "git",
                 "url": "https://github.com/egulias/EmailValidator.git",
-                "reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff"
+                "reference": "ebaaf5be6c0286928352e054f2d5125608e5405e"
             },
             "dist": {
                 "type": "zip",
-                "url": "https://api.github.com/repos/egulias/EmailValidator/zipball/3a85486b709bc384dae8eb78fb2eec649bdb64ff",
+                "url": "https://api.github.com/repos/egulias/EmailValidator/zipball/ebaaf5be6c0286928352e054f2d5125608e5405e",
-                "reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff",
+                "reference": "ebaaf5be6c0286928352e054f2d5125608e5405e",
                 "shasum": ""
             },
             "require": {
@@ -1416,8 +1416,8 @@
                 "symfony/polyfill-intl-idn": "^1.26"
             },
             "require-dev": {
-                "phpunit/phpunit": "^9.5.27",
+                "phpunit/phpunit": "^10.2",
-                "vimeo/psalm": "^4.30"
+                "vimeo/psalm": "^5.12"
             },
             "suggest": {
                 "ext-intl": "PHP Internationalization Libraries are required to use the SpoofChecking validation"
@@ -1453,7 +1453,7 @@
             ],
             "support": {
                 "issues": "https://github.com/egulias/EmailValidator/issues",
-                "source": "https://github.com/egulias/EmailValidator/tree/4.0.1"
+                "source": "https://github.com/egulias/EmailValidator/tree/4.0.2"
             },
             "funding": [
                 {
@@ -1461,7 +1461,7 @@
                     "type": "github"
                 }
             ],
-            "time": "2023-01-14T14:17:03+00:00"
+            "time": "2023-10-06T06:47:41+00:00"
         },
         {
             "name": "jdenticon/jdenticon",
@@ -7604,6 +7604,47 @@
                 "source": "https://github.com/webmozarts/assert/tree/1.11.0"
             },
             "time": "2022-06-03T18:03:27+00:00"
+        },
+        {
+            "name": "yggverse/scrapeer",
+            "version": "0.5.4",
+            "source": {
+                "type": "git",
+                "url": "https://github.com/YGGverse/scrapeer.git",
+                "reference": "3b9d15c9706a844bf1428812bb938370e907b4f3"
+            },
+            "dist": {
+                "type": "zip",
+                "url": "https://api.github.com/repos/YGGverse/scrapeer/zipball/3b9d15c9706a844bf1428812bb938370e907b4f3",
+                "reference": "3b9d15c9706a844bf1428812bb938370e907b4f3",
+                "shasum": ""
+            },
+            "type": "library",
+            "autoload": {
+                "psr-4": {
+                    "Yggverse\\Scrapeer\\": "src/"
+                }
+            },
+            "notification-url": "https://packagist.org/downloads/",
+            "license": [
+                "CC-BY-SA-3.0"
+            ],
+            "authors": [
+                {
+                    "name": "medariox",
+                    "homepage": "https://github.com/medariox/scrapeer"
+                },
+                {
+                    "name": "YGGverse",
+                    "homepage": "https://github.com/YGGverse/scrapeer"
+                }
+            ],
+            "description": "Essential PHP library that scrapes HTTP(S) and UDP trackers for torrent information",
+            "support": {
+                "issues": "https://github.com/YGGverse/scrapeer/issues",
+                "source": "https://github.com/YGGverse/scrapeer/tree/0.5.4"
+            },
+            "time": "2023-10-09T01:47:30+00:00"
         }
     ],
     "packages-dev": [

src/Controller/TorrentController.php

@@ -72,8 +72,8 @@ class TorrentController extends AbstractController
                     'peers'     => (int) $torrent->getPeers(),
                     'leechers'  => (int) $torrent->getLeechers(),
                 ],
-                'locales'   => $torrentService->findLastTorrentLocales($torrent->getId()),
+                'locales'   => $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()),
-                'sensitive' => $torrentService->findLastTorrentSensitive($torrent->getId())->isValue(),
+                'sensitive' => $torrentService->findLastTorrentSensitiveByTorrentId($torrent->getId())->isValue(),
                 'download'  =>
                 [
                     'file' =>
@@ -346,7 +346,7 @@ class TorrentController extends AbstractController
         // Otherwise, get latest available
         else
         {
-            if ($torrentLocales = $torrentService->findLastTorrentLocales($torrent->getId()))
+            if ($torrentLocales = $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()))
             {
                 $torrentLocalesCurrent['userId'] = $torrentLocales->getUserId();
 
@@ -367,7 +367,7 @@ class TorrentController extends AbstractController
 
         // Init edition history
         $editions = [];
-        foreach ($torrentService->findTorrentLocales($torrent->getId()) as $torrentLocalesEdition)
+        foreach ($torrentService->findTorrentLocalesByTorrentId($torrent->getId()) as $torrentLocalesEdition)
         {
             $editions[] =
             [
@@ -652,7 +652,7 @@ class TorrentController extends AbstractController
         }
         else
         {
-            if ($torrentSensitive = $torrentService->findLastTorrentSensitive($request->get('torrentId')))
+            if ($torrentSensitive = $torrentService->findLastTorrentSensitiveByTorrentId($request->get('torrentId')))
             {
                 $torrentSensitiveCurrent =
                 [
@@ -675,7 +675,7 @@ class TorrentController extends AbstractController
 
         // Init edition history
         $editions = [];
-        foreach ($torrentService->findTorrentSensitive($torrent->getId()) as $torrentSensitiveEdition)
+        foreach ($torrentService->findTorrentSensitiveByTorrentId($torrent->getId()) as $torrentSensitiveEdition)
         {
             $editions[] =
             [
@@ -1088,4 +1088,26 @@ class TorrentController extends AbstractController
             $file->getMagnetLink()
         );
     }
+
+    // Tools
+    #[Route(
+        '/crontab/scrape',
+        methods:
+        [
+            'GET'
+        ]
+    )]
+    public function scrape(
+        Request $request,
+        TranslatorInterface $translator,
+        TorrentService $torrentService,
+    ): Response
+    {
+        $torrentService->scrapeTorrentQueue(
+            explode('|', $this->getParameter('app.trackers'))
+        );
+
+        // Render response
+        return new Response(); // @TODO
+    }
 }

src/Entity/Torrent.php

@@ -20,6 +20,9 @@ class Torrent
     #[ORM\Column]
     private ?int $added = null;
 
+    #[ORM\Column(nullable: true)]
+    private ?int $scraped = null;
+
     #[ORM\Column]
     private ?bool $approved = null;
 
@@ -71,6 +74,18 @@ class Torrent
         return $this;
     }
 
+    public function getScraped(): ?int
+    {
+        return $this->scraped;
+    }
+
+    public function setScraped(int $scraped): static
+    {
+        $this->scraped = $scraped;
+
+        return $this;
+    }
+
     public function getKeywords(): ?string
     {
         return $this->keywords;

src/Repository/TorrentRepository.php

@@ -30,4 +30,14 @@ class TorrentRepository extends ServiceEntityRepository
             ->getOneOrNullResult()
         ;
     }
+
+    public function getTorrentScrapeQueue(): ?Torrent
+    {
+        return $this->createQueryBuilder('t')
+            ->orderBy('t.scraped', 'ASC') // same to ts.added
+            ->setMaxResults(1)
+            ->getQuery()
+            ->getOneOrNullResult()
+        ;
+    }
 }

src/Service/TorrentService.php

@@ -36,6 +36,80 @@ class TorrentService
     }
 
     // Tools
+    public function scrapeTorrentQueue(
+        array $trackers = []
+    ): void
+    {
+        // Init Scraper
+        $scraper = new \Yggverse\Scrapeer\Scraper();
+
+        if ($torrent = $this->getTorrentScrapeQueue())
+        {
+            // Get file
+            if (!$file = $this->readTorrentFileByTorrentId($torrent->getId()))
+            {
+                // @TODO
+                throw new \Exception(
+                    $translator->trans('File not found')
+                );
+            }
+
+            // Get info hashes
+            $hashes = [];
+
+            if ($hash = $file->getInfoHashV1(false))
+            {
+                $hashes[] = $hash;
+            }
+
+            if ($hash = $file->getInfoHashV2(false))
+            {
+                $hashes[] = $hash;
+            }
+
+            // Get scrape
+            if ($hashes && $trackers)
+            {
+                // Update scrape info
+                if ($results = $scraper->scrape($hashes, $trackers, null, 1))
+                {
+                    foreach ($results as $result)
+                    {
+                        if (isset($result['seeders']))
+                        {
+                            $torrent->setSeeders(
+                                (int) $result['seeders']
+                            );
+                        }
+
+                        if (isset($result['completed']))
+                        {
+                            $torrent->setPeers(
+                                (int) $result['completed']
+                            );
+                        }
+
+                        if (isset($result['leechers']))
+                        {
+                            $torrent->setLeechers(
+                                (int) $result['leechers']
+                            );
+                        }
+                    }
+                }
+            }
+
+            // Update time scraped
+            $torrent->setScraped(
+                time()
+            );
+
+            // Save results to DB
+            $this->entityManagerInterface->persist($torrent);
+            $this->entityManagerInterface->flush();
+        }
+    }
+
     public function readTorrentFileByFilepath(
         string $filepath
     ): ?\Rhilip\Bencode\TorrentFile
@@ -188,6 +262,13 @@ class TorrentService
         return $torrent;
     }
 
+    public function getTorrentScrapeQueue(): ?Torrent
+    {
+        return $this->entityManagerInterface
+                    ->getRepository(Torrent::class)
+                    ->getTorrentScrapeQueue();
+    }
+
     // Torrent locale
     public function getTorrentLocales(int $id): ?TorrentLocales
     {