YGGverse
/
YGGstate
mirror of https://github.com/YGGverse/YGGstate.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

implement tracker, peers JSON API

yggdrasil-0.4.7
ghost 1 year ago
parent
f03da2cd3d
commit
606a6f1cac
5 changed files with 210 additions and 172 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      .gitignore
  2. 7
      README.md
  3. 13
      src/config/app.php.example
  4. 347
      src/crontab/crawler.php
  5. 12
      src/public/search.php

3
.gitignore vendored
Unescape View File

@ -6,5 +6,8 @@
/src/config/app.php /src/config/app.php
/src/public/api/peers.json
/src/public/api/trackers.json
.ftpignore .ftpignore
composer.lock composer.lock

7
README.md
Unescape View File

@ -54,7 +54,12 @@ git checkout -b my-pr-branch-name
#### Roadmap #### Roadmap
* [ ] Federated tracking API API
* [x] trackers
* [x] peers
Crawler
* [ ] federated model
#### Donate to contributors #### Donate to contributors

13
src/config/app.php.example
Unescape View File

@ -72,12 +72,13 @@ define('WEBSITE_PEER_REMOTE_PAGINATION_LIMIT', 20);
define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5); define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5);
// Trackers // API
define('TRACKER_PUBLIC_PEERS', (object) define('API_PEERS', (object)
[ [
'tls://94.140.114.241:4708', 'tls://94.140.114.241:4708' => WEBSITE_URL,
// ... // ...
]); ]
);
// Crawler // Crawler
define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 128); define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 128);

347
src/crontab/crawler.php
Unescape View File

@ -129,82 +129,104 @@ try {
exit; exit;
} }
// Collect connected peers // Delete cache
@unlink(__DIR__ . '/../public/api/peers.json');
@unlink(__DIR__ . '/../public/api/trackers.json');
// Update peers
if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) { if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { if ($handle = fopen(__DIR__ . '/../public/api/peers.json', 'w+')) {
fwrite($handle, json_encode($connectedPeers));
fclose($handle);
}
try { } else {
$db->beginTransaction(); exit;
}
// Init peer // Update trackers
if ($dbPeer = $db->findPeer($connectedPeerAddress)) { if (API_PEERS) {
$dbPeerId = $dbPeer->peerId; if ($handle = fopen(__DIR__ . '/../public/api/trackers.json', 'w+')) {
fwrite($handle, json_encode(API_PEERS));
fclose($handle);
}
} else { } else {
if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) { exit;
}
$debug['yggdrasil']['peer']['total']['insert']++; // @TODO merge peers data from remote trackers
}
}
// Init peer session // Collect connected peers
if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) { foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
$dbPeerSessionId = $dbLastPeerSession->peerSessionId; try {
// If remote session uptime < than stored, register new one $db->beginTransaction();
if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) {
if ($dbPeerSessionId = $db->addPeerSession($dbPeerId, // Init peer
round($connectedPeerInfo->uptime), if ($dbPeer = $db->findPeer($connectedPeerAddress)) {
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
$debug['yggdrasil']['peer']['session']['total']['insert']++; $dbPeerId = $dbPeer->peerId;
}
} else { } else {
$debug['yggdrasil']['peer']['session']['total']['update'] += if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) {
$db->updatePeerSession($dbLastPeerSession->peerSessionId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time());
}
} else { $debug['yggdrasil']['peer']['total']['insert']++;
}
}
// Init peer session
if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) {
$dbPeerSessionId = $dbLastPeerSession->peerSessionId;
// If remote session uptime < than stored, register new one
if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) {
if ($dbPeerSessionId = $db->addPeerSession($dbPeerId, if ($dbPeerSessionId = $db->addPeerSession($dbPeerId,
round($connectedPeerInfo->uptime), round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent, $connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd, $connectedPeerInfo->bytes_recvd,
time())) { time())) {
$debug['yggdrasil']['peer']['session']['total']['insert']++; $debug['yggdrasil']['peer']['session']['total']['insert']++;
} }
} else {
$debug['yggdrasil']['peer']['session']['total']['update'] +=
$db->updatePeerSession($dbLastPeerSession->peerSessionId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time());
} }
// Init peer coordinate } else {
if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) {
$dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId; if ($dbPeerSessionId = $db->addPeerSession($dbPeerId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
// Peer have changed it port, init new coordinate $debug['yggdrasil']['peer']['session']['total']['insert']++;
if ($dbPeerCoordinate->port != $connectedPeerInfo->port) { }
}
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { // Init peer coordinate
if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++; $dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId;
}
}
} else { // Peer have changed it port, init new coordinate
if ($dbPeerCoordinate->port != $connectedPeerInfo->port) {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
@ -212,203 +234,210 @@ if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
} }
} }
// Init peer coordinate route } else {
$dbCoords = [];
foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) { if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port;
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
} }
}
// Compare remote / local route, create new on changed // Init peer coordinate route
if ($dbCoords !== $connectedPeerInfo->coords) { $dbCoords = [];
foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) {
$dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port;
}
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { // Compare remote / local route, create new on changed
if ($dbCoords !== $connectedPeerInfo->coords) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++; if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
}
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
foreach ($connectedPeerInfo->coords as $level => $port) { foreach ($connectedPeerInfo->coords as $level => $port) {
if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) { if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) {
$debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++; $debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++;
}
} }
} }
}
// Init peer remote // Init peer remote
if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) { if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
// Init peer scheme // Init peer scheme
if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) { if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) {
$dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId; $dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId;
} else { } else {
if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) { if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) {
$debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++; $debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++;
}
} }
}
// Init peer host // Init peer host
if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) { if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) {
$dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId; $dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId;
} else { } else {
if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) { if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) {
$debug['yggdrasil']['peer']['remote']['host']['total']['insert']++; $debug['yggdrasil']['peer']['remote']['host']['total']['insert']++;
}
} }
}
// Init peer port // Init peer port
if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) { if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) {
$dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId; $dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId;
} else { } else {
if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) { if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) {
$debug['yggdrasil']['peer']['remote']['port']['total']['insert']++; $debug['yggdrasil']['peer']['remote']['port']['total']['insert']++;
}
} }
}
// Init geo data // Init geo data
/// Country /// Country
$countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode; $countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode;
$countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name; $countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name;
$dbGeoCountryId = null; $dbGeoCountryId = null;
if (!empty($countryIsoCode) && !empty($countryName)) { if (!empty($countryIsoCode) && !empty($countryName)) {
if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) { if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) {
$dbGeoCountryId = $dbGeoCountry->geoCountryId; $dbGeoCountryId = $dbGeoCountry->geoCountryId;
} else { } else {
if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) { if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) {
$debug['yggdrasil']['geo']['country']['total']['insert']++; $debug['yggdrasil']['geo']['country']['total']['insert']++;
}
} }
} }
}
/// City /// City
$cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name; $cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name;
$dbGeoCityId = null; $dbGeoCityId = null;
if (!empty($cityName)) { if (!empty($cityName)) {
if ($dbGeoCity = $db->findGeoCityByName($cityName)) { if ($dbGeoCity = $db->findGeoCityByName($cityName)) {
$dbGeoCityId = $dbGeoCity->geoCityId; $dbGeoCityId = $dbGeoCity->geoCityId;
} else { } else {
if ($dbGeoCityId = $db->addGeoCity($cityName)) { if ($dbGeoCityId = $db->addGeoCity($cityName)) {
$debug['yggdrasil']['geo']['city']['total']['insert']++; $debug['yggdrasil']['geo']['city']['total']['insert']++;
}
} }
} }
}
/// Coordinate /// Coordinate
$latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude; $latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude;
$longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude; $longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude;
$dbGeoCoordinateId = null; $dbGeoCoordinateId = null;
if (!empty($latitude) && !empty($longitude)) { if (!empty($latitude) && !empty($longitude)) {
if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId; $dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId;
} else { } else {
if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$debug['yggdrasil']['geo']['coordinate']['total']['insert']++; $debug['yggdrasil']['geo']['coordinate']['total']['insert']++;
}
} }
} }
}
/// Geo /// Geo
$dbGeoId = null; $dbGeoId = null;
if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$dbGeoId = $dbGeo->geoId; $dbGeoId = $dbGeo->geoId;
} else { } else {
if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) { if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$debug['yggdrasil']['geo']['total']['insert']++; $debug['yggdrasil']['geo']['total']['insert']++;
}
} }
}
// Init peer remote // Init peer remote
if ($dbPeerRemote = $db->findPeerRemote($dbPeerId, if ($dbPeerRemote = $db->findPeerRemote($dbPeerId,
$dbGeoId, $dbGeoId,
$dbPeerRemoteSchemeId, $dbPeerRemoteSchemeId,
$dbPeerRemoteHostId, $dbPeerRemoteHostId,
$dbPeerRemotePortId)) { $dbPeerRemotePortId)) {
$dbPeerRemoteId = $dbPeerRemote->peerRemoteId; $dbPeerRemoteId = $dbPeerRemote->peerRemoteId;
} else { } else {
if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId, if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId,
$dbGeoId, $dbGeoId,
$dbPeerRemoteSchemeId, $dbPeerRemoteSchemeId,
$dbPeerRemoteHostId, $dbPeerRemoteHostId,
$dbPeerRemotePortId, $dbPeerRemotePortId,
time())) { time())) {
$debug['yggdrasil']['peer']['remote']['total']['insert']++; $debug['yggdrasil']['peer']['remote']['total']['insert']++;
}
} }
}
// If something went wrong with URL parse, skip next operations for this peer // If something went wrong with URL parse, skip next operations for this peer
} else { } else {
$db->rollBack(); $db->rollBack();
continue; continue;
} }
// Init peer connection // Init peer connection
if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) { if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) {
if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) { if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) {
$debug['yggdrasil']['peer']['connection']['total']['insert']++; $debug['yggdrasil']['peer']['connection']['total']['insert']++;
}
} }
}
$debug['yggdrasil']['peer']['total']['online']++; $debug['yggdrasil']['peer']['total']['online']++;
$db->commit(); $db->commit();
} catch(Exception $e) { } catch(Exception $e) {
$db->rollBack(); $db->rollBack();
var_dump($e); var_dump($e);
break; break;
}
} }
} }

12
src/public/search.php
Unescape View File

@ -235,10 +235,10 @@ $results = $sphinx->searchPeers($requestQuery,
<?php } ?> <?php } ?>
</td> </td>
<td colspan="5" class="text-right"> <td colspan="5" class="text-right">
<?php if (TRACKER_PUBLIC_PEERS) { ?> <?php if (API_PEERS) { ?>
<div class="margin-top-8"><?php echo _('trackers') ?></div> <div class="margin-top-8"><?php echo _('trackers') ?></div>
<?php foreach (TRACKER_PUBLIC_PEERS as $address) { ?> <?php foreach (API_PEERS as $tracker => $website) { ?>
<div><?php echo $address ?></div> <div><?php echo $tracker ?></div>
<?php } ?> <?php } ?>
<?php } ?> <?php } ?>
</td> </td>
@ -249,10 +249,10 @@ $results = $sphinx->searchPeers($requestQuery,
<tr> <tr>
<td class="text-center"> <td class="text-center">
<div class="margin-y-8"><?php echo _('not found') ?></div> <div class="margin-y-8"><?php echo _('not found') ?></div>
<?php if (TRACKER_PUBLIC_PEERS) { ?> <?php if (API_PEERS) { ?>
<div class="margin-y-8"><?php echo _('get yourself tracked by connection') ?></div> <div class="margin-y-8"><?php echo _('get yourself tracked by connection') ?></div>
<?php foreach (TRACKER_PUBLIC_PEERS as $address) { ?> <?php foreach (API_PEERS as $tracker => $website) { ?>
<div class="margin-y-8"><?php echo $address ?></div> <div class="margin-y-8"><?php echo $tracker ?></div>
<?php } ?> <?php } ?>
<?php } ?> <?php } ?>
</td> </td>

Write Preview
Loading…
Cancel
Save