YGGverse
/
YGGtracker
mirror of https://github.com/YGGverse/YGGtracker.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

implement torrent scrape queue

main
ghost 1 year ago
parent
d2f7fff24e
commit
1b722812e8
7 changed files with 194 additions and 24 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 2
      README.md
  2. 3
      composer.json
  3. 73
      composer.lock
  4. 34
      src/Controller/TorrentController.php
  5. 15
      src/Entity/Torrent.php
  6. 10
      src/Repository/TorrentRepository.php
  7. 81
      src/Service/TorrentService.php

2
README.md
Unescape View File

@ -47,7 +47,7 @@ git checkout -b my-pr-branch-name @@ -47,7 +47,7 @@ git checkout -b my-pr-branch-name
#### Components
* [SVG icons](https://icons.getbootstrap.com)
* [Scrapper](https://github.com/medariox/scrapeer)
* [Scrapper](https://github.com/medariox/scrapeer) / [Composer Edition](https://github.com/YGGverse/scrapeer)
* [Bencode Library](https://github.com/Rhilip/Bencode)
* [Identicons](https://github.com/dmester/jdenticon-php)

3
composer.json
Unescape View File

@ -47,7 +47,8 @@ @@ -47,7 +47,8 @@
"twig/extra-bundle": "^2.12|^3.0",
"twig/intl-extra": "^3.7",
"twig/string-extra": "^3.7",
"twig/twig": "^2.12|^3.0"
"twig/twig": "^2.12|^3.0",
"yggverse/scrapeer": "^0.5.4"
},
"config": {
"allow-plugins": {

73
composer.lock generated
Unescape View File

@ -4,7 +4,7 @@ @@ -4,7 +4,7 @@
"Read more about it at https://getcomposer.org/doc/01-basic-usage.md#installing-dependencies",
"This file is @generated automatically"
],
"content-hash": "d46bb514c4109b10e7327ef9f913c11a",
"content-hash": "3770ffcd80695bc10a22f8ece4f68d1f",
"packages": [
{
"name": "doctrine/annotations",
@ -177,16 +177,16 @@ @@ -177,16 +177,16 @@
},
{
"name": "doctrine/collections",
"version": "2.1.3",
"version": "2.1.4",
"source": {
"type": "git",
"url": "https://github.com/doctrine/collections.git",
"reference": "3023e150f90a38843856147b58190aa8b46cc155"
"reference": "72328a11443a0de79967104ad36ba7b30bded134"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/doctrine/collections/zipball/3023e150f90a38843856147b58190aa8b46cc155",
"reference": "3023e150f90a38843856147b58190aa8b46cc155",
"url": "https://api.github.com/repos/doctrine/collections/zipball/72328a11443a0de79967104ad36ba7b30bded134",
"reference": "72328a11443a0de79967104ad36ba7b30bded134",
"shasum": ""
},
"require": {
@ -194,7 +194,7 @@ @@ -194,7 +194,7 @@
"php": "^8.1"
},
"require-dev": {
"doctrine/coding-standard": "^10.0",
"doctrine/coding-standard": "^12",
"ext-json": "*",
"phpstan/phpstan": "^1.8",
"phpstan/phpstan-phpunit": "^1.0",
@ -243,7 +243,7 @@ @@ -243,7 +243,7 @@
],
"support": {
"issues": "https://github.com/doctrine/collections/issues",
"source": "https://github.com/doctrine/collections/tree/2.1.3"
"source": "https://github.com/doctrine/collections/tree/2.1.4"
},
"funding": [
{
@ -259,7 +259,7 @@ @@ -259,7 +259,7 @@
"type": "tidelift"
}
],
"time": "2023-07-06T15:15:36+00:00"
"time": "2023-10-03T09:22:33+00:00"
},
{
"name": "doctrine/common",
@ -1398,16 +1398,16 @@ @@ -1398,16 +1398,16 @@
},
{
"name": "egulias/email-validator",
"version": "4.0.1",
"version": "4.0.2",
"source": {
"type": "git",
"url": "https://github.com/egulias/EmailValidator.git",
"reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff"
"reference": "ebaaf5be6c0286928352e054f2d5125608e5405e"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/egulias/EmailValidator/zipball/3a85486b709bc384dae8eb78fb2eec649bdb64ff",
"reference": "3a85486b709bc384dae8eb78fb2eec649bdb64ff",
"url": "https://api.github.com/repos/egulias/EmailValidator/zipball/ebaaf5be6c0286928352e054f2d5125608e5405e",
"reference": "ebaaf5be6c0286928352e054f2d5125608e5405e",
"shasum": ""
},
"require": {
@ -1416,8 +1416,8 @@ @@ -1416,8 +1416,8 @@
"symfony/polyfill-intl-idn": "^1.26"
},
"require-dev": {
"phpunit/phpunit": "^9.5.27",
"vimeo/psalm": "^4.30"
"phpunit/phpunit": "^10.2",
"vimeo/psalm": "^5.12"
},
"suggest": {
"ext-intl": "PHP Internationalization Libraries are required to use the SpoofChecking validation"
@ -1453,7 +1453,7 @@ @@ -1453,7 +1453,7 @@
],
"support": {
"issues": "https://github.com/egulias/EmailValidator/issues",
"source": "https://github.com/egulias/EmailValidator/tree/4.0.1"
"source": "https://github.com/egulias/EmailValidator/tree/4.0.2"
},
"funding": [
{
@ -1461,7 +1461,7 @@ @@ -1461,7 +1461,7 @@
"type": "github"
}
],
"time": "2023-01-14T14:17:03+00:00"
"time": "2023-10-06T06:47:41+00:00"
},
{
"name": "jdenticon/jdenticon",
@ -7604,6 +7604,47 @@ @@ -7604,6 +7604,47 @@
"source": "https://github.com/webmozarts/assert/tree/1.11.0"
},
"time": "2022-06-03T18:03:27+00:00"
},
{
"name": "yggverse/scrapeer",
"version": "0.5.4",
"source": {
"type": "git",
"url": "https://github.com/YGGverse/scrapeer.git",
"reference": "3b9d15c9706a844bf1428812bb938370e907b4f3"
},
"dist": {
"type": "zip",
"url": "https://api.github.com/repos/YGGverse/scrapeer/zipball/3b9d15c9706a844bf1428812bb938370e907b4f3",
"reference": "3b9d15c9706a844bf1428812bb938370e907b4f3",
"shasum": ""
},
"type": "library",
"autoload": {
"psr-4": {
"Yggverse\\Scrapeer\\": "src/"
}
},
"notification-url": "https://packagist.org/downloads/",
"license": [
"CC-BY-SA-3.0"
],
"authors": [
{
"name": "medariox",
"homepage": "https://github.com/medariox/scrapeer"
},
{
"name": "YGGverse",
"homepage": "https://github.com/YGGverse/scrapeer"
}
],
"description": "Essential PHP library that scrapes HTTP(S) and UDP trackers for torrent information",
"support": {
"issues": "https://github.com/YGGverse/scrapeer/issues",
"source": "https://github.com/YGGverse/scrapeer/tree/0.5.4"
},
"time": "2023-10-09T01:47:30+00:00"
}
],
"packages-dev": [

34
src/Controller/TorrentController.php
Unescape View File

@ -72,8 +72,8 @@ class TorrentController extends AbstractController @@ -72,8 +72,8 @@ class TorrentController extends AbstractController
'peers' => (int) $torrent->getPeers(),
'leechers' => (int) $torrent->getLeechers(),
],
'locales' => $torrentService->findLastTorrentLocales($torrent->getId()),
'sensitive' => $torrentService->findLastTorrentSensitive($torrent->getId())->isValue(),
'locales' => $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()),
'sensitive' => $torrentService->findLastTorrentSensitiveByTorrentId($torrent->getId())->isValue(),
'download' =>
[
'file' =>
@ -346,7 +346,7 @@ class TorrentController extends AbstractController @@ -346,7 +346,7 @@ class TorrentController extends AbstractController
// Otherwise, get latest available
else
{
if ($torrentLocales = $torrentService->findLastTorrentLocales($torrent->getId()))
if ($torrentLocales = $torrentService->findLastTorrentLocalesByTorrentId($torrent->getId()))
{
$torrentLocalesCurrent['userId'] = $torrentLocales->getUserId();
@ -367,7 +367,7 @@ class TorrentController extends AbstractController @@ -367,7 +367,7 @@ class TorrentController extends AbstractController
// Init edition history
$editions = [];
foreach ($torrentService->findTorrentLocales($torrent->getId()) as $torrentLocalesEdition)
foreach ($torrentService->findTorrentLocalesByTorrentId($torrent->getId()) as $torrentLocalesEdition)
{
$editions[] =
[
@ -652,7 +652,7 @@ class TorrentController extends AbstractController @@ -652,7 +652,7 @@ class TorrentController extends AbstractController
}
else
{
if ($torrentSensitive = $torrentService->findLastTorrentSensitive($request->get('torrentId')))
if ($torrentSensitive = $torrentService->findLastTorrentSensitiveByTorrentId($request->get('torrentId')))
{
$torrentSensitiveCurrent =
[
@ -675,7 +675,7 @@ class TorrentController extends AbstractController @@ -675,7 +675,7 @@ class TorrentController extends AbstractController
// Init edition history
$editions = [];
foreach ($torrentService->findTorrentSensitive($torrent->getId()) as $torrentSensitiveEdition)
foreach ($torrentService->findTorrentSensitiveByTorrentId($torrent->getId()) as $torrentSensitiveEdition)
{
$editions[] =
[
@ -1088,4 +1088,26 @@ class TorrentController extends AbstractController @@ -1088,4 +1088,26 @@ class TorrentController extends AbstractController
$file->getMagnetLink()
);
}
// Tools
#[Route(
'/crontab/scrape',
methods:
[
'GET'
]
)]
public function scrape(
Request $request,
TranslatorInterface $translator,
TorrentService $torrentService,
): Response
{
$torrentService->scrapeTorrentQueue(
explode('|', $this->getParameter('app.trackers'))
);
// Render response
return new Response(); // @TODO
}
}

15
src/Entity/Torrent.php
Unescape View File

@ -20,6 +20,9 @@ class Torrent @@ -20,6 +20,9 @@ class Torrent
#[ORM\Column]
private ?int $added = null;
#[ORM\Column(nullable: true)]
private ?int $scraped = null;
#[ORM\Column]
private ?bool $approved = null;
@ -71,6 +74,18 @@ class Torrent @@ -71,6 +74,18 @@ class Torrent
return $this;
}
public function getScraped(): ?int
{
return $this->scraped;
}
public function setScraped(int $scraped): static
{
$this->scraped = $scraped;
return $this;
}
public function getKeywords(): ?string
{
return $this->keywords;

10
src/Repository/TorrentRepository.php
Unescape View File

@ -30,4 +30,14 @@ class TorrentRepository extends ServiceEntityRepository @@ -30,4 +30,14 @@ class TorrentRepository extends ServiceEntityRepository
->getOneOrNullResult()
;
}
public function getTorrentScrapeQueue(): ?Torrent
{
return $this->createQueryBuilder('t')
->orderBy('t.scraped', 'ASC') // same to ts.added
->setMaxResults(1)
->getQuery()
->getOneOrNullResult()
;
}
}

81
src/Service/TorrentService.php
Unescape View File

@ -36,6 +36,80 @@ class TorrentService @@ -36,6 +36,80 @@ class TorrentService
}
// Tools
public function scrapeTorrentQueue(
array $trackers = []
): void
{
// Init Scraper
$scraper = new \Yggverse\Scrapeer\Scraper();
if ($torrent = $this->getTorrentScrapeQueue())
{
// Get file
if (!$file = $this->readTorrentFileByTorrentId($torrent->getId()))
{
// @TODO
throw new \Exception(
$translator->trans('File not found')
);
}
// Get info hashes
$hashes = [];
if ($hash = $file->getInfoHashV1(false))
{
$hashes[] = $hash;
}
if ($hash = $file->getInfoHashV2(false))
{
$hashes[] = $hash;
}
// Get scrape
if ($hashes && $trackers)
{
// Update scrape info
if ($results = $scraper->scrape($hashes, $trackers, null, 1))
{
foreach ($results as $result)
{
if (isset($result['seeders']))
{
$torrent->setSeeders(
(int) $result['seeders']
);
}
if (isset($result['completed']))
{
$torrent->setPeers(
(int) $result['completed']
);
}
if (isset($result['leechers']))
{
$torrent->setLeechers(
(int) $result['leechers']
);
}
}
}
}
// Update time scraped
$torrent->setScraped(
time()
);
// Save results to DB
$this->entityManagerInterface->persist($torrent);
$this->entityManagerInterface->flush();
}
}
public function readTorrentFileByFilepath(
string $filepath
): ?\Rhilip\Bencode\TorrentFile
@ -188,6 +262,13 @@ class TorrentService @@ -188,6 +262,13 @@ class TorrentService
return $torrent;
}
public function getTorrentScrapeQueue(): ?Torrent
{
return $this->entityManagerInterface
->getRepository(Torrent::class)
->getTorrentScrapeQueue();
}
// Torrent locale
public function getTorrentLocales(int $id): ?TorrentLocales
{

Write Preview
Loading…
Cancel
Save