From 606a6f1cac7f62dcd615ec53f6474449c5d784fb Mon Sep 17 00:00:00 2001 From: ghost Date: Thu, 24 Aug 2023 15:42:46 +0300 Subject: [PATCH] implement tracker, peers JSON API --- .gitignore | 3 + README.md | 7 +- src/config/app.php.example | 13 +- src/crontab/crawler.php | 347 ++++++++++++++++++++----------------- src/public/search.php | 12 +- 5 files changed, 210 insertions(+), 172 deletions(-) diff --git a/.gitignore b/.gitignore index f35fda1..ef75ec6 100644 --- a/.gitignore +++ b/.gitignore @@ -6,5 +6,8 @@ /src/config/app.php +/src/public/api/peers.json +/src/public/api/trackers.json + .ftpignore composer.lock diff --git a/README.md b/README.md index 652b326..cc8b8d2 100644 --- a/README.md +++ b/README.md @@ -54,7 +54,12 @@ git checkout -b my-pr-branch-name #### Roadmap - * [ ] Federated tracking API +API + * [x] trackers + * [x] peers + +Crawler + * [ ] federated model #### Donate to contributors diff --git a/src/config/app.php.example b/src/config/app.php.example index b281d7b..54dbeca 100644 --- a/src/config/app.php.example +++ b/src/config/app.php.example @@ -72,12 +72,13 @@ define('WEBSITE_PEER_REMOTE_PAGINATION_LIMIT', 20); define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5); -// Trackers -define('TRACKER_PUBLIC_PEERS', (object) -[ - 'tls://94.140.114.241:4708', - // ... -]); +// API +define('API_PEERS', (object) + [ + 'tls://94.140.114.241:4708' => WEBSITE_URL, + // ... + ] +); // Crawler define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 128); \ No newline at end of file diff --git a/src/crontab/crawler.php b/src/crontab/crawler.php index 957b5f8..20020cb 100644 --- a/src/crontab/crawler.php +++ b/src/crontab/crawler.php @@ -129,82 +129,104 @@ try { exit; } -// Collect connected peers +// Delete cache +@unlink(__DIR__ . '/../public/api/peers.json'); +@unlink(__DIR__ . '/../public/api/trackers.json'); + +// Update peers if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) { - foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { + if ($handle = fopen(__DIR__ . '/../public/api/peers.json', 'w+')) { + fwrite($handle, json_encode($connectedPeers)); + fclose($handle); + } - try { +} else { - $db->beginTransaction(); + exit; +} - // Init peer - if ($dbPeer = $db->findPeer($connectedPeerAddress)) { +// Update trackers +if (API_PEERS) { - $dbPeerId = $dbPeer->peerId; + if ($handle = fopen(__DIR__ . '/../public/api/trackers.json', 'w+')) { + fwrite($handle, json_encode(API_PEERS)); + fclose($handle); + } - } else { +} else { - if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) { + exit; +} - $debug['yggdrasil']['peer']['total']['insert']++; - } - } +// @TODO merge peers data from remote trackers - // Init peer session - if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) { +// Collect connected peers +foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { - $dbPeerSessionId = $dbLastPeerSession->peerSessionId; + try { - // If remote session uptime < than stored, register new one - if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) { + $db->beginTransaction(); - if ($dbPeerSessionId = $db->addPeerSession($dbPeerId, - round($connectedPeerInfo->uptime), - $connectedPeerInfo->bytes_sent, - $connectedPeerInfo->bytes_recvd, - time())) { + // Init peer + if ($dbPeer = $db->findPeer($connectedPeerAddress)) { - $debug['yggdrasil']['peer']['session']['total']['insert']++; - } + $dbPeerId = $dbPeer->peerId; - } else { + } else { - $debug['yggdrasil']['peer']['session']['total']['update'] += - $db->updatePeerSession($dbLastPeerSession->peerSessionId, - round($connectedPeerInfo->uptime), - $connectedPeerInfo->bytes_sent, - $connectedPeerInfo->bytes_recvd, - time()); - } + if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) { - } else { + $debug['yggdrasil']['peer']['total']['insert']++; + } + } + + // Init peer session + if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) { + + $dbPeerSessionId = $dbLastPeerSession->peerSessionId; + + // If remote session uptime < than stored, register new one + if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) { if ($dbPeerSessionId = $db->addPeerSession($dbPeerId, - round($connectedPeerInfo->uptime), - $connectedPeerInfo->bytes_sent, - $connectedPeerInfo->bytes_recvd, - time())) { + round($connectedPeerInfo->uptime), + $connectedPeerInfo->bytes_sent, + $connectedPeerInfo->bytes_recvd, + time())) { $debug['yggdrasil']['peer']['session']['total']['insert']++; } + + } else { + + $debug['yggdrasil']['peer']['session']['total']['update'] += + $db->updatePeerSession($dbLastPeerSession->peerSessionId, + round($connectedPeerInfo->uptime), + $connectedPeerInfo->bytes_sent, + $connectedPeerInfo->bytes_recvd, + time()); } - // Init peer coordinate - if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) { + } else { - $dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId; + if ($dbPeerSessionId = $db->addPeerSession($dbPeerId, + round($connectedPeerInfo->uptime), + $connectedPeerInfo->bytes_sent, + $connectedPeerInfo->bytes_recvd, + time())) { - // Peer have changed it port, init new coordinate - if ($dbPeerCoordinate->port != $connectedPeerInfo->port) { + $debug['yggdrasil']['peer']['session']['total']['insert']++; + } + } - if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { + // Init peer coordinate + if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) { - $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; - } - } + $dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId; - } else { + // Peer have changed it port, init new coordinate + if ($dbPeerCoordinate->port != $connectedPeerInfo->port) { if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { @@ -212,203 +234,210 @@ if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) { } } - // Init peer coordinate route - $dbCoords = []; - foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) { - $dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port; + } else { + + if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { + + $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; } + } - // Compare remote / local route, create new on changed - if ($dbCoords !== $connectedPeerInfo->coords) { + // Init peer coordinate route + $dbCoords = []; + foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) { + $dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port; + } - if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { + // Compare remote / local route, create new on changed + if ($dbCoords !== $connectedPeerInfo->coords) { - $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; - } + if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { + + $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; + } - foreach ($connectedPeerInfo->coords as $level => $port) { + foreach ($connectedPeerInfo->coords as $level => $port) { - if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) { + if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) { - $debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++; - } + $debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++; } } + } - // Init peer remote - if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) { + // Init peer remote + if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) { - // Init peer scheme - if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) { + // Init peer scheme + if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) { - $dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId; + $dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId; - } else { + } else { - if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) { + if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) { - $debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++; - } + $debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++; } + } - // Init peer host - if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) { + // Init peer host + if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) { - $dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId; + $dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId; - } else { + } else { - if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) { + if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) { - $debug['yggdrasil']['peer']['remote']['host']['total']['insert']++; - } + $debug['yggdrasil']['peer']['remote']['host']['total']['insert']++; } + } - // Init peer port - if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) { + // Init peer port + if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) { - $dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId; + $dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId; - } else { + } else { - if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) { + if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) { - $debug['yggdrasil']['peer']['remote']['port']['total']['insert']++; - } + $debug['yggdrasil']['peer']['remote']['port']['total']['insert']++; } + } - // Init geo data + // Init geo data - /// Country - $countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode; - $countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name; + /// Country + $countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode; + $countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name; - $dbGeoCountryId = null; + $dbGeoCountryId = null; - if (!empty($countryIsoCode) && !empty($countryName)) { + if (!empty($countryIsoCode) && !empty($countryName)) { - if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) { + if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) { - $dbGeoCountryId = $dbGeoCountry->geoCountryId; + $dbGeoCountryId = $dbGeoCountry->geoCountryId; - } else { + } else { - if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) { + if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) { - $debug['yggdrasil']['geo']['country']['total']['insert']++; - } + $debug['yggdrasil']['geo']['country']['total']['insert']++; } } + } - /// City - $cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name; + /// City + $cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name; - $dbGeoCityId = null; + $dbGeoCityId = null; - if (!empty($cityName)) { + if (!empty($cityName)) { - if ($dbGeoCity = $db->findGeoCityByName($cityName)) { + if ($dbGeoCity = $db->findGeoCityByName($cityName)) { - $dbGeoCityId = $dbGeoCity->geoCityId; + $dbGeoCityId = $dbGeoCity->geoCityId; - } else { + } else { - if ($dbGeoCityId = $db->addGeoCity($cityName)) { + if ($dbGeoCityId = $db->addGeoCity($cityName)) { - $debug['yggdrasil']['geo']['city']['total']['insert']++; - } + $debug['yggdrasil']['geo']['city']['total']['insert']++; } } + } - /// Coordinate - $latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude; - $longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude; + /// Coordinate + $latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude; + $longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude; - $dbGeoCoordinateId = null; + $dbGeoCoordinateId = null; - if (!empty($latitude) && !empty($longitude)) { + if (!empty($latitude) && !empty($longitude)) { - if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, - $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { + if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, + $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { - $dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId; + $dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId; - } else { + } else { - if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, - $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { + if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, + $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { - $debug['yggdrasil']['geo']['coordinate']['total']['insert']++; - } + $debug['yggdrasil']['geo']['coordinate']['total']['insert']++; } } + } - /// Geo - $dbGeoId = null; + /// Geo + $dbGeoId = null; - if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { + if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { - $dbGeoId = $dbGeo->geoId; + $dbGeoId = $dbGeo->geoId; - } else { + } else { - if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { + if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { - $debug['yggdrasil']['geo']['total']['insert']++; - } + $debug['yggdrasil']['geo']['total']['insert']++; } + } - // Init peer remote - if ($dbPeerRemote = $db->findPeerRemote($dbPeerId, - $dbGeoId, - $dbPeerRemoteSchemeId, - $dbPeerRemoteHostId, - $dbPeerRemotePortId)) { + // Init peer remote + if ($dbPeerRemote = $db->findPeerRemote($dbPeerId, + $dbGeoId, + $dbPeerRemoteSchemeId, + $dbPeerRemoteHostId, + $dbPeerRemotePortId)) { - $dbPeerRemoteId = $dbPeerRemote->peerRemoteId; + $dbPeerRemoteId = $dbPeerRemote->peerRemoteId; - } else { + } else { - if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId, - $dbGeoId, - $dbPeerRemoteSchemeId, - $dbPeerRemoteHostId, - $dbPeerRemotePortId, - time())) { + if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId, + $dbGeoId, + $dbPeerRemoteSchemeId, + $dbPeerRemoteHostId, + $dbPeerRemotePortId, + time())) { - $debug['yggdrasil']['peer']['remote']['total']['insert']++; - } + $debug['yggdrasil']['peer']['remote']['total']['insert']++; } + } - // If something went wrong with URL parse, skip next operations for this peer - } else { + // If something went wrong with URL parse, skip next operations for this peer + } else { - $db->rollBack(); + $db->rollBack(); - continue; - } + continue; + } - // Init peer connection - if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) { + // Init peer connection + if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) { - if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) { + if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) { - $debug['yggdrasil']['peer']['connection']['total']['insert']++; - } + $debug['yggdrasil']['peer']['connection']['total']['insert']++; } + } - $debug['yggdrasil']['peer']['total']['online']++; + $debug['yggdrasil']['peer']['total']['online']++; - $db->commit(); + $db->commit(); - } catch(Exception $e) { + } catch(Exception $e) { - $db->rollBack(); + $db->rollBack(); - var_dump($e); + var_dump($e); - break; - } + break; } } diff --git a/src/public/search.php b/src/public/search.php index d63c87a..97cd2ee 100644 --- a/src/public/search.php +++ b/src/public/search.php @@ -235,10 +235,10 @@ $results = $sphinx->searchPeers($requestQuery, - +
- -
+ $website) { ?> +
@@ -249,10 +249,10 @@ $results = $sphinx->searchPeers($requestQuery,
- +
- -
+ $website) { ?> +