From 8190fc1914e850e932ce2f09159e31112daeb561 Mon Sep 17 00:00:00 2001 From: ghost Date: Mon, 23 Oct 2023 21:16:40 +0300 Subject: [PATCH] implement FTP storage for wanted feature #27 --- .env | 24 ++- config/services.yaml | 2 + src/Controller/TorrentController.php | 211 ++++++++++++++++++++++++++- src/Service/TorrentService.php | 147 +++++++++---------- 4 files changed, 296 insertions(+), 88 deletions(-) diff --git a/.env b/.env index 188fef3..9b83a59 100644 --- a/.env +++ b/.env @@ -46,24 +46,46 @@ MESSENGER_TRANSPORT_DSN=doctrine://default?auto_setup=0 ###< symfony/crowdin-translation-provider ### # YGGtracker + +# Application version, used for API and media cache APP_VERSION='2.1.0' +# Application name APP_NAME=YGGtracker +# Default locale APP_LOCALE=en + +# Supported locales for interface and content filters APP_LOCALES=en|cs|eo|fr|ka|de|he|it|lv|pl|pt|ru|es|uk +# Items per page on pagination APP_PAGINATION=10 +# Default application theme APP_THEME=default + +# Additional themes, stored in /src/templates, /public/asset APP_THEMES=default +# Default sensitive status for new users APP_SENSITIVE=0 + +# Default approved status for new users APP_APPROVED=0 + +# Default Yggdrasil filters status for new users APP_YGGDRASIL=1 +# Build-in trackers append to downloads APP_TRACKERS=http://[201:23b4:991a:634d:8359:4521:5576:15b7]:2023/announce|http://[200:1e2f:e608:eb3a:2bf:1e62:87ba:e2f7]/announce|http://[316:c51a:62a3:8b9::5]/announce +# List of crawlers where ignored in actions and activity features APP_CRAWLERS=201:23b4:991a:634d:8359:4521:5576:15b7|30a:5fad::e -APP_TORRENT_FILE_SIZE_MAX=1024000 \ No newline at end of file +# Max torrent filesize for uploads +APP_TORRENT_FILE_SIZE_MAX=1024000 + +# Store wanted torrent files in /app/var/ftp by /app/crontab/torrent/scrape/{key} +APP_TORRENT_WANTED_FTP_ENABLED=1 +APP_TORRENT_WANTED_FTP_APPROVED_ONLY=1 \ No newline at end of file diff --git a/config/services.yaml b/config/services.yaml index 7e673e1..e2cb3f6 100644 --- a/config/services.yaml +++ b/config/services.yaml @@ -18,6 +18,8 @@ parameters: app.approved: '%env(APP_APPROVED)%' app.yggdrasil: '%env(APP_YGGDRASIL)%' app.torrent.size.max: '%env(APP_TORRENT_FILE_SIZE_MAX)%' + app.torrent.wanted.ftp.enabled: '%env(APP_TORRENT_WANTED_FTP_ENABLED)%' + app.torrent.wanted.ftp.approved: '%env(APP_TORRENT_WANTED_FTP_APPROVED_ONLY)%' services: # default configuration for services in *this* file diff --git a/src/Controller/TorrentController.php b/src/Controller/TorrentController.php index 48395de..cde50c6 100644 --- a/src/Controller/TorrentController.php +++ b/src/Controller/TorrentController.php @@ -1858,10 +1858,11 @@ class TorrentController extends AbstractController $response->headers->set( 'Content-Disposition', sprintf( - 'attachment; filename="%s.%s.torrent";', + 'attachment; filename="%s#%s.%s.torrent";', mb_strtolower( $this->getParameter('app.name') ), + $torrent->getId(), mb_strtolower( $file->getName() ) @@ -1978,10 +1979,8 @@ class TorrentController extends AbstractController $response->headers->set( 'Content-Disposition', sprintf( - 'attachment; filename="%s.wanted.%s.torrent";', - mb_strtolower( - $this->getParameter('app.name') - ), + 'attachment; filename="wanted#%s.%s.torrent";', + $torrent->getId(), mb_strtolower( $file->getName() ) @@ -2102,10 +2101,208 @@ class TorrentController extends AbstractController TorrentService $torrentService, ): Response { - $torrentService->scrapeTorrentQueue( - explode('|', $this->getParameter('app.trackers')) + // Init Scraper + $scraper = new \Yggverse\Scrapeer\Scraper(); + + // Get next torrent in scrape queue + if (!$torrent = $torrentService->getTorrentScrapeQueue()) + { + throw $this->createNotFoundException(); + } + + // Get file + if (!$file = $torrentService->readTorrentFileByTorrentId($torrent->getId())) + { + throw $this->createNotFoundException(); + } + + // Filter yggdrasil trackers + $file = $this->filterYggdrasil($file, true); + + // Get trackers list + $trackers = []; + + if ($announce = $file->getAnnounce()) + { + $trackers[] = $announce; + } + + if ($announceList = $file->getAnnounceList()) + { + if (isset($announceList[0])) + { + foreach ($announceList[0] as $value) + { + $trackers[] = $value; + } + } + + if (isset($announceList[1])) + { + foreach ($announceList[1] as $value) + { + $trackers[] = $value; + } + } + } + + $trackers = array_unique($trackers); + + // Get info hashes + $hashes = []; + + if ($hash = $file->getInfoHashV1(false)) + { + $hashes[] = $hash; + } + + if ($hash = $file->getInfoHashV2(false)) + { + $hashes[] = $hash; + } + + // Get scrape + $seeders = 0; + $peers = 0; + $leechers = 0; + + if ($hashes && $trackers) + { + // Update scrape info + if ($results = $scraper->scrape($hashes, $trackers, null, 1)) + { + foreach ($results as $result) + { + if (isset($result['seeders'])) + { + $seeders = $seeders + (int) $result['seeders']; + } + + if (isset($result['completed'])) + { + $peers = $peers + (int) $result['completed']; + } + + if (isset($result['leechers'])) + { + $leechers = $leechers + (int) $result['leechers']; + } + } + } + } + + // Update DB + $torrentService->updateTorrentScrape( + $torrent->getId(), + $seeders, + $peers, + $leechers ); + // Update torrent wanted storage if enabled + if ($this->getParameter('app.torrent.wanted.ftp.enabled') === '1') + { + // Add wanted file + if ($leechers && !$seeders) + { + if ($this->getParameter('app.torrent.wanted.ftp.approved') === '0' || + ($this->getParameter('app.torrent.wanted.ftp.approved') === '1' && $torrent->isApproved())) + { + /// All + $torrentService->copyToFtpStorage( + $torrent->getId(), + sprintf( + '/torrents/wanted/all/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + + /// Sensitive + if ($torrent->isSensitive()) + { + $torrentService->copyToFtpStorage( + $torrent->getId(), + sprintf( + '/torrents/wanted/sensitive/yes/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + } + + else + { + $torrentService->copyToFtpStorage( + $torrent->getId(), + sprintf( + '/torrents/wanted/sensitive/no/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + } + + /// Locals + foreach ($torrent->getLocales() as $locale) + { + $torrentService->copyToFtpStorage( + $torrent->getId(), + sprintf( + '/torrents/wanted/locale/%s/wanted#%s.%s.torrent', + $locale, + $torrent->getId(), + $file->getName() + ) + ); + } + } + } + + // Remove not wanted files + else + { + /// All + $torrentService->removeFromFtpStorage( + sprintf( + '/torrents/wanted/all/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + + /// Sensitive + $torrentService->removeFromFtpStorage( + sprintf( + '/torrents/wanted/sensitive/yes/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + + $torrentService->removeFromFtpStorage( + sprintf( + '/torrents/wanted/sensitive/no/wanted#%s.%s.torrent', + $torrent->getId(), + $file->getName() + ) + ); + + /// Locals + foreach (explode('|', $this->getParameter('app.locales')) as $locale) + { + $torrentService->removeFromFtpStorage( + sprintf( + '/torrents/wanted/locale/%s/wanted#%s.%s.torrent', + $locale, + $torrent->getId(), + $file->getName() + ) + ); + } + } + } + // Render response return new Response(); // @TODO } diff --git a/src/Service/TorrentService.php b/src/Service/TorrentService.php index c34642f..5e2c4c3 100644 --- a/src/Service/TorrentService.php +++ b/src/Service/TorrentService.php @@ -36,86 +36,6 @@ class TorrentService } // Tools - public function scrapeTorrentQueue( - array $trackers = [] - ): void - { - // Init Scraper - $scraper = new \Yggverse\Scrapeer\Scraper(); - - if ($torrent = $this->getTorrentScrapeQueue()) - { - // Init default values - $seeders = 0; - $peers = 0; - $leechers = 0; - - // Get file - if ($file = $this->readTorrentFileByTorrentId($torrent->getId())) - { - // Get info hashes - $hashes = []; - - if ($hash = $file->getInfoHashV1(false)) - { - $hashes[] = $hash; - } - - if ($hash = $file->getInfoHashV2(false)) - { - $hashes[] = $hash; - } - - // Get scrape - if ($hashes && $trackers) - { - // Update scrape info - if ($results = $scraper->scrape($hashes, $trackers, null, 1)) - { - foreach ($results as $result) - { - if (isset($result['seeders'])) - { - $seeders = $seeders + (int) $result['seeders']; - } - - if (isset($result['completed'])) - { - $peers = $peers + (int) $result['completed']; - } - - if (isset($result['leechers'])) - { - $leechers = $leechers + (int) $result['leechers']; - } - } - } - } - } - - // Update torrent scrape - $torrent->setSeeders( - $seeders - ); - - $torrent->setPeers( - $peers - ); - - $torrent->setLeechers( - $leechers - ); - - $torrent->setScraped( - time() - ); - - // Save results to DB - $this->entityManagerInterface->persist($torrent); - $this->entityManagerInterface->flush(); - } - } - public function readTorrentFileByFilepath( string $filepath ): ?\Rhilip\Bencode\TorrentFile @@ -210,6 +130,15 @@ class TorrentService ); } + public function getFtpFilepathByFilename(string $filename): string + { + return sprintf( + '%s/var/ftp/%s', + $this->kernelInterface->getProjectDir(), + $filename + ); + } + public function getTorrentContributors(Torrent $torrent): array { $contributors = []; @@ -229,6 +158,34 @@ class TorrentService return array_unique($contributors); } + public function copyToFtpStorage( + int $torrentId, + string $filename + ): void + { + $filesystem = new Filesystem(); + $filesystem->copy( + $this->getStorageFilepathByTorrentId( + $torrentId + ), + $this->getFtpFilepathByFilename( + $filename + ) + ); + } + + public function removeFromFtpStorage( + string $filename + ): void + { + $filesystem = new Filesystem(); + $filesystem->remove( + $this->getFtpFilepathByFilename( + $filename + ) + ); + } + public function add( string $filepath, int $userId, @@ -450,6 +407,36 @@ class TorrentService } } + public function updateTorrentScrape( + int $torrentId, + int $seeders, + int $peers, + int $leechers + ): void + { + if ($torrent = $this->getTorrent($torrentId)) + { + $torrent->setSeeders( + $seeders + ); + + $torrent->setPeers( + $peers + ); + + $torrent->setLeechers( + $leechers + ); + + $torrent->setScraped( + time() + ); + + $this->entityManagerInterface->persist($torrent); + $this->entityManagerInterface->flush(); + } + } + public function reindexTorrentKeywordsAll(): void { foreach ($this->entityManagerInterface