YGGverse
/
YGGstate
mirror of https://github.com/YGGverse/YGGstate.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

implement tracker, peers JSON API

yggdrasil-0.4.7
ghost 9 months ago
parent
f03da2cd3d
commit
606a6f1cac
5 changed files with 210 additions and 172 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      .gitignore
  2. 7
      README.md
  3. 13
      src/config/app.php.example
  4. 347
      src/crontab/crawler.php
  5. 12
      src/public/search.php

3
.gitignore vendored
Unescape View File

@ -6,5 +6,8 @@ @@ -6,5 +6,8 @@
/src/config/app.php
/src/public/api/peers.json
/src/public/api/trackers.json
.ftpignore
composer.lock

7
README.md
Unescape View File

@ -54,7 +54,12 @@ git checkout -b my-pr-branch-name @@ -54,7 +54,12 @@ git checkout -b my-pr-branch-name
#### Roadmap
* [ ] Federated tracking API
API
* [x] trackers
* [x] peers
Crawler
* [ ] federated model
#### Donate to contributors

13
src/config/app.php.example
Unescape View File

@ -72,12 +72,13 @@ define('WEBSITE_PEER_REMOTE_PAGINATION_LIMIT', 20); @@ -72,12 +72,13 @@ define('WEBSITE_PEER_REMOTE_PAGINATION_LIMIT', 20);
define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5);
// Trackers
define('TRACKER_PUBLIC_PEERS', (object)
[
'tls://94.140.114.241:4708',
// ...
]);
// API
define('API_PEERS', (object)
[
'tls://94.140.114.241:4708' => WEBSITE_URL,
// ...
]
);
// Crawler
define('CRAWL_STOP_DISK_QUOTA_MB_LEFT', 128);

347
src/crontab/crawler.php
Unescape View File

@ -129,82 +129,104 @@ try { @@ -129,82 +129,104 @@ try {
exit;
}
// Collect connected peers
// Delete cache
@unlink(__DIR__ . '/../public/api/peers.json');
@unlink(__DIR__ . '/../public/api/trackers.json');
// Update peers
if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
if ($handle = fopen(__DIR__ . '/../public/api/peers.json', 'w+')) {
fwrite($handle, json_encode($connectedPeers));
fclose($handle);
}
try {
} else {
$db->beginTransaction();
exit;
}
// Init peer
if ($dbPeer = $db->findPeer($connectedPeerAddress)) {
// Update trackers
if (API_PEERS) {
$dbPeerId = $dbPeer->peerId;
if ($handle = fopen(__DIR__ . '/../public/api/trackers.json', 'w+')) {
fwrite($handle, json_encode(API_PEERS));
fclose($handle);
}
} else {
} else {
if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) {
exit;
}
$debug['yggdrasil']['peer']['total']['insert']++;
}
}
// @TODO merge peers data from remote trackers
// Init peer session
if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) {
// Collect connected peers
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
$dbPeerSessionId = $dbLastPeerSession->peerSessionId;
try {
// If remote session uptime < than stored, register new one
if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) {
$db->beginTransaction();
if ($dbPeerSessionId = $db->addPeerSession($dbPeerId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
// Init peer
if ($dbPeer = $db->findPeer($connectedPeerAddress)) {
$debug['yggdrasil']['peer']['session']['total']['insert']++;
}
$dbPeerId = $dbPeer->peerId;
} else {
} else {
$debug['yggdrasil']['peer']['session']['total']['update'] +=
$db->updatePeerSession($dbLastPeerSession->peerSessionId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time());
}
if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) {
} else {
$debug['yggdrasil']['peer']['total']['insert']++;
}
}
// Init peer session
if ($dbLastPeerSession = $db->findLastPeerSessionByPeerId($dbPeerId)) {
$dbPeerSessionId = $dbLastPeerSession->peerSessionId;
// If remote session uptime < than stored, register new one
if ($connectedPeerInfo->uptime < $dbLastPeerSession->uptime) {
if ($dbPeerSessionId = $db->addPeerSession($dbPeerId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
$debug['yggdrasil']['peer']['session']['total']['insert']++;
}
} else {
$debug['yggdrasil']['peer']['session']['total']['update'] +=
$db->updatePeerSession($dbLastPeerSession->peerSessionId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time());
}
// Init peer coordinate
if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) {
} else {
$dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId;
if ($dbPeerSessionId = $db->addPeerSession($dbPeerId,
round($connectedPeerInfo->uptime),
$connectedPeerInfo->bytes_sent,
$connectedPeerInfo->bytes_recvd,
time())) {
// Peer have changed it port, init new coordinate
if ($dbPeerCoordinate->port != $connectedPeerInfo->port) {
$debug['yggdrasil']['peer']['session']['total']['insert']++;
}
}
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
// Init peer coordinate
if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
}
$dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId;
} else {
// Peer have changed it port, init new coordinate
if ($dbPeerCoordinate->port != $connectedPeerInfo->port) {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
@ -212,203 +234,210 @@ if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) { @@ -212,203 +234,210 @@ if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
}
}
// Init peer coordinate route
$dbCoords = [];
foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) {
$dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port;
} else {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
}
// Compare remote / local route, create new on changed
if ($dbCoords !== $connectedPeerInfo->coords) {
// Init peer coordinate route
$dbCoords = [];
foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) {
$dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port;
}
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
// Compare remote / local route, create new on changed
if ($dbCoords !== $connectedPeerInfo->coords) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
foreach ($connectedPeerInfo->coords as $level => $port) {
foreach ($connectedPeerInfo->coords as $level => $port) {
if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) {
if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) {
$debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++;
}
$debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++;
}
}
}
// Init peer remote
if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
// Init peer remote
if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
// Init peer scheme
if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) {
// Init peer scheme
if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) {
$dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId;
$dbPeerRemoteSchemeId = $dbPeerRemoteScheme->peerRemoteSchemeId;
} else {
} else {
if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) {
if ($dbPeerRemoteSchemeId = $db->addPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme, time())) {
$debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++;
}
$debug['yggdrasil']['peer']['remote']['scheme']['total']['insert']++;
}
}
// Init peer host
if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) {
// Init peer host
if ($dbPeerRemoteHost = $db->findPeerRemoteHost($connectedPeerRemoteUrl->host->name)) {
$dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId;
$dbPeerRemoteHostId = $dbPeerRemoteHost->peerRemoteHostId;
} else {
} else {
if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) {
if ($dbPeerRemoteHostId = $db->addPeerRemoteHost($connectedPeerRemoteUrl->host->name, time())) {
$debug['yggdrasil']['peer']['remote']['host']['total']['insert']++;
}
$debug['yggdrasil']['peer']['remote']['host']['total']['insert']++;
}
}
// Init peer port
if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) {
// Init peer port
if ($dbPeerRemotePort = $db->findPeerRemotePort($connectedPeerRemoteUrl->host->port)) {
$dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId;
$dbPeerRemotePortId = $dbPeerRemotePort->peerRemotePortId;
} else {
} else {
if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) {
if ($dbPeerRemotePortId = $db->addPeerRemotePort($connectedPeerRemoteUrl->host->port, time())) {
$debug['yggdrasil']['peer']['remote']['port']['total']['insert']++;
}
$debug['yggdrasil']['peer']['remote']['port']['total']['insert']++;
}
}
// Init geo data
// Init geo data
/// Country
$countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode;
$countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name;
/// Country
$countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode;
$countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name;
$dbGeoCountryId = null;
$dbGeoCountryId = null;
if (!empty($countryIsoCode) && !empty($countryName)) {
if (!empty($countryIsoCode) && !empty($countryName)) {
if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) {
if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) {
$dbGeoCountryId = $dbGeoCountry->geoCountryId;
$dbGeoCountryId = $dbGeoCountry->geoCountryId;
} else {
} else {
if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) {
if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) {
$debug['yggdrasil']['geo']['country']['total']['insert']++;
}
$debug['yggdrasil']['geo']['country']['total']['insert']++;
}
}
}
/// City
$cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name;
/// City
$cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name;
$dbGeoCityId = null;
$dbGeoCityId = null;
if (!empty($cityName)) {
if (!empty($cityName)) {
if ($dbGeoCity = $db->findGeoCityByName($cityName)) {
if ($dbGeoCity = $db->findGeoCityByName($cityName)) {
$dbGeoCityId = $dbGeoCity->geoCityId;
$dbGeoCityId = $dbGeoCity->geoCityId;
} else {
} else {
if ($dbGeoCityId = $db->addGeoCity($cityName)) {
if ($dbGeoCityId = $db->addGeoCity($cityName)) {
$debug['yggdrasil']['geo']['city']['total']['insert']++;
}
$debug['yggdrasil']['geo']['city']['total']['insert']++;
}
}
}
/// Coordinate
$latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude;
$longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude;
/// Coordinate
$latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude;
$longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude;
$dbGeoCoordinateId = null;
$dbGeoCoordinateId = null;
if (!empty($latitude) && !empty($longitude)) {
if (!empty($latitude) && !empty($longitude)) {
if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId;
$dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId;
} else {
} else {
if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$debug['yggdrasil']['geo']['coordinate']['total']['insert']++;
}
$debug['yggdrasil']['geo']['coordinate']['total']['insert']++;
}
}
}
/// Geo
$dbGeoId = null;
/// Geo
$dbGeoId = null;
if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$dbGeoId = $dbGeo->geoId;
$dbGeoId = $dbGeo->geoId;
} else {
} else {
if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$debug['yggdrasil']['geo']['total']['insert']++;
}
$debug['yggdrasil']['geo']['total']['insert']++;
}
}
// Init peer remote
if ($dbPeerRemote = $db->findPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId)) {
// Init peer remote
if ($dbPeerRemote = $db->findPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId)) {
$dbPeerRemoteId = $dbPeerRemote->peerRemoteId;
$dbPeerRemoteId = $dbPeerRemote->peerRemoteId;
} else {
} else {
if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId,
time())) {
if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId,
time())) {
$debug['yggdrasil']['peer']['remote']['total']['insert']++;
}
$debug['yggdrasil']['peer']['remote']['total']['insert']++;
}
}
// If something went wrong with URL parse, skip next operations for this peer
} else {
// If something went wrong with URL parse, skip next operations for this peer
} else {
$db->rollBack();
$db->rollBack();
continue;
}
continue;
}
// Init peer connection
if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) {
// Init peer connection
if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) {
if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) {
if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) {
$debug['yggdrasil']['peer']['connection']['total']['insert']++;
}
$debug['yggdrasil']['peer']['connection']['total']['insert']++;
}
}
$debug['yggdrasil']['peer']['total']['online']++;
$debug['yggdrasil']['peer']['total']['online']++;
$db->commit();
$db->commit();
} catch(Exception $e) {
} catch(Exception $e) {
$db->rollBack();
$db->rollBack();
var_dump($e);
var_dump($e);
break;
}
break;
}
}

12
src/public/search.php
Unescape View File

@ -235,10 +235,10 @@ $results = $sphinx->searchPeers($requestQuery, @@ -235,10 +235,10 @@ $results = $sphinx->searchPeers($requestQuery,
<?php } ?>
</td>
<td colspan="5" class="text-right">
<?php if (TRACKER_PUBLIC_PEERS) { ?>
<?php if (API_PEERS) { ?>
<div class="margin-top-8"><?php echo _('trackers') ?></div>
<?php foreach (TRACKER_PUBLIC_PEERS as $address) { ?>
<div><?php echo $address ?></div>
<?php foreach (API_PEERS as $tracker => $website) { ?>
<div><?php echo $tracker ?></div>
<?php } ?>
<?php } ?>
</td>
@ -249,10 +249,10 @@ $results = $sphinx->searchPeers($requestQuery, @@ -249,10 +249,10 @@ $results = $sphinx->searchPeers($requestQuery,
<tr>
<td class="text-center">
<div class="margin-y-8"><?php echo _('not found') ?></div>
<?php if (TRACKER_PUBLIC_PEERS) { ?>
<?php if (API_PEERS) { ?>
<div class="margin-y-8"><?php echo _('get yourself tracked by connection') ?></div>
<?php foreach (TRACKER_PUBLIC_PEERS as $address) { ?>
<div class="margin-y-8"><?php echo $address ?></div>
<?php foreach (API_PEERS as $tracker => $website) { ?>
<div class="margin-y-8"><?php echo $tracker ?></div>
<?php } ?>
<?php } ?>
</td>

Write Preview
Loading…
Cancel
Save