YGGverse/YGGstate
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGstate.git synced 2025-03-12 05:11:11 +00:00
Code Issues Projects Releases Wiki Activity

add yggdrasil v 0.5.1 support, remove deprecated constructions #5

Browse Source
This commit is contained in:
ghost 2023-10-29 06:09:57 +02:00
parent 3138486aef
commit 4c5e6eed75
10 changed files with 49 additions and 198 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
composer.json
View File

@ -4,7 +4,7 @@
"type": "project",
"require": {
"php": ">=8.1",
"yggverse/yggdrasilctl": ">=0.1.0",
"yggverse/yggdrasilctl": ">=1.0.0",
"yggverse/parser": ">=0.1.0",
"yggverse/graph": ">=0.1.0",
"yggverse/cache": ">=0.3.0",

BIN
database/yggstate.mwb
View File

Binary file not shown.

17
example/environment /sphinx.conf
View File

@ -15,13 +15,6 @@ source peer : yggstate
SELECT `peer`.`peerId`, \
`peer`.`address` AS `peerAddress`, \
`peer`.`key` AS `peerKey`, \
(SELECT GROUP_CONCAT(DISTINCT `peerCoordinate`.`port`) \
FROM `peerCoordinate` \
WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `peerCoordinatePort`, \
(SELECT GROUP_CONCAT(DISTINCT `peerCoordinateRoute`.`port`) \
FROM `peerCoordinateRoute` \
JOIN `peerCoordinate` ON (`peerCoordinate`.`peerCoordinateId` = `peerCoordinateRoute`.`peerCoordinateId`) \
WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `peerCoordinateRoute`, \
(SELECT GROUP_CONCAT(DISTINCT `peerRemoteScheme`.`name`) \
FROM `peerRemote` \
JOIN `peerRemoteScheme` ON (`peerRemoteScheme`.`peerRemoteSchemeId` = `peerRemote`.`peerRemoteSchemeId`) \
@ -56,8 +49,6 @@ source peer : yggstate
sql_field_string = peerAddress
sql_field_string = peerKey
sql_field_string = peerCoordinatePort
sql_field_string = peerCoordinateRoute
sql_field_string = peerRemoteScheme
sql_field_string = peerRemoteHost
sql_field_string = peerRemotePort
@ -68,12 +59,12 @@ source peer : yggstate
index peer
{
source = peer
path = /var/lib/sphinxsearch/data/peer
source = peer
path = /var/lib/sphinxsearch/data/peer
}
indexer
{
mem_limit = 256M
lemmatizer_cache = 256M
mem_limit = 256M
lemmatizer_cache = 256M
}

5
src/config/app.php.example
View File

@ -75,11 +75,10 @@ define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5);
// API
define('API_PEER_FIELDS', (array)
[
'key',
'address',
'bytes_recvd',
'bytes_sent',
'coords',
'key',
'port',
'uptime',
// ...
]

83
src/crontab/crawler.php
View File

@ -65,16 +65,6 @@ $debug = [
]
]
],
'coordinate' => [
'total' => [
'insert' => 0,
],
'route' => [
'total' => [
'insert' => 0,
]
]
],
'connection' => [
'total' => [
'insert' => 0,
@ -134,8 +124,14 @@ try {
@unlink(__DIR__ . '/../public/api/trackers.json');
// Update peers
if (!$connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers())
$connectedPeersDebug = [];
if (false === $connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers($connectedPeersDebug))
{
var_dump(
$connectedPeersDebug
);
exit;
}
@ -143,13 +139,13 @@ if (!$connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers())
/// Remove remote addresses
$jsonPeers = [];
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo)
foreach ($connectedPeers as $connectedPeerInfo)
{
foreach ($connectedPeerInfo as $connectedPeerInfoKey => $connectedPeerInfoValue)
foreach ($connectedPeerInfo as $key => $value)
{
if (in_array($connectedPeerInfoKey, (array) API_PEER_FIELDS))
if (in_array($key, (array) API_PEER_FIELDS))
{
$jsonPeers[$connectedPeerAddress][$connectedPeerInfoKey] = $connectedPeerInfoValue;
$jsonPeers[$connectedPeerInfo->address][$key] = $value;
}
}
}
@ -171,20 +167,20 @@ if ($handle = fopen(__DIR__ . '/../public/api/trackers.json', 'w+'))
// @TODO merge peers data from remote trackers
// Collect connected peers
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
foreach ($connectedPeers as $connectedPeerInfo) {
try {
$db->beginTransaction();
// Init peer
if ($dbPeer = $db->findPeer($connectedPeerAddress)) {
if ($dbPeer = $db->findPeer($connectedPeerInfo->address)) {
$dbPeerId = $dbPeer->peerId;
} else {
if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) {
if ($dbPeerId = $db->addPeer($connectedPeerInfo->address, $connectedPeerInfo->key, time())) {
$debug['yggdrasil']['peer']['total']['insert']++;
}
@ -229,51 +225,6 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
}
}
// Init peer coordinate
if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) {
$dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId;
// Peer have changed it port, init new coordinate
if ($dbPeerCoordinate->port != $connectedPeerInfo->port) {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
}
} else {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
}
// Init peer coordinate route
$dbCoords = [];
foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) {
$dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port;
}
// Compare remote / local route, create new on changed
if ($dbCoords !== $connectedPeerInfo->coords) {
if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) {
$debug['yggdrasil']['peer']['coordinate']['total']['insert']++;
}
foreach ($connectedPeerInfo->coords as $level => $port) {
if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) {
$debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++;
}
}
}
// Init peer remote
if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
@ -375,7 +326,7 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
} else {
if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$debug['yggdrasil']['geo']['coordinate']['total']['insert']++;
}
@ -428,9 +379,9 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
}
// Init peer connection
if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) {
if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId)) {
if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) {
if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, time())) {
$debug['yggdrasil']['peer']['connection']['total']['insert']++;
}

90
src/library/mysql.php
View File

@ -457,84 +457,32 @@ class MySQL {
}
// Peer connection
public function addPeerConnection(int $peerSessionId, int $peerRemoteId, int $peerCoordinateId, int $timeAdded) {
public function addPeerConnection(int $peerSessionId, int $peerRemoteId, int $timeAdded) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `peerConnection` SET `peerSessionId` = ?,
`peerRemoteId` = ?,
`peerCoordinateId` = ?,
`timeAdded` = ?');
$query->execute([$peerSessionId, $peerRemoteId, $peerCoordinateId, $timeAdded]);
$query->execute([$peerSessionId, $peerRemoteId, $timeAdded]);
return $this->_db->lastInsertId();
}
public function findPeerConnection(int $peerSessionId, int $peerRemoteId, int $peerCoordinateId) {
public function findPeerConnection(int $peerSessionId, int $peerRemoteId) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `peerConnection`
WHERE `peerSessionId` = ? AND `peerRemoteId` = ? AND `peerCoordinateId` = ?
WHERE `peerSessionId` = ? AND `peerRemoteId` = ?
LIMIT 1');
$query->execute([$peerSessionId, $peerRemoteId, $peerCoordinateId]);
$query->execute([$peerSessionId, $peerRemoteId]);
return $query->fetch();
}
// Peer coordinate
// https://github.com/matrix-org/pinecone/wiki/2.-Spanning-Tree#coordinates
public function addPeerCoordinate(int $peerId, int $port, int $timeAdded) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `peerCoordinate` SET `peerId` = ?,
`port` = ?,
`timeAdded` = ?');
$query->execute([$peerId, $port, $timeAdded]);
return $this->_db->lastInsertId();
}
public function addPeerCoordinateRoute(int $peerCoordinateId, int $level, int $port) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `peerCoordinateRoute` SET `peerCoordinateId` = ?,
`level` = ?,
`port` = ?');
$query->execute([$peerCoordinateId, $level, $port]);
return $this->_db->lastInsertId();
}
public function findLastPeerCoordinateByPeerId(int $peerId) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `peerCoordinate` WHERE `peerId` = ? ORDER BY `peerCoordinateId` DESC LIMIT 1');
$query->execute([$peerId]);
return $query->fetch();
}
public function findPeerCoordinateRouteByCoordinateId(int $peerCoordinateId) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `peerCoordinateRoute` WHERE `peerCoordinateId` = ? ORDER BY `level` ASC');
$query->execute([$peerCoordinateId]);
return $query->fetchAll();
}
// Analytics
public function getPeerFirstByTimeAdded() {
@ -703,15 +651,6 @@ class MySQL {
`peerRemoteHost`.`name` AS `remoteHost`,
`peerRemotePort`.`name` AS `remotePort`,
`peerCoordinate`.`port` AS `connectionPort`,
(
SELECT GROUP_CONCAT(`port` SEPARATOR ' → ')
FROM `peerCoordinateRoute`
WHERE `peerCoordinateRoute`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId`
ORDER BY `peerCoordinateRoute`.`level` ASC
) AS `route`,
CONCAT
(
IF (`peerRemotePort`.`name` IS NOT NULL,
@ -728,7 +667,6 @@ class MySQL {
JOIN `peerSession` ON (`peerSession`.`peerSessionId` = `peerConnection`.`peerSessionId`)
JOIN `peerRemote` ON (`peerRemote`.`peerRemoteId` = `peerConnection`.`peerRemoteId`)
JOIN `peerCoordinate` ON (`peerCoordinate`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId`)
JOIN `peerRemoteScheme` ON (`peerRemoteScheme`.`peerRemoteSchemeId` = `peerRemote`.`peerRemoteSchemeId`)
JOIN `peerRemoteHost` ON (`peerRemoteHost`.`peerRemoteHostId` = `peerRemote`.`peerRemoteHostId`)
@ -774,8 +712,7 @@ class MySQL {
ROUND(AVG(`peerSession`.`uptime`)) AS `uptimeAvg`,
(SELECT COUNT(*) FROM `peerSession` WHERE `peerSession`.`peerId` = `peer`.`peerId`) AS `sessionTotal`,
(SELECT COUNT(*) FROM `peerRemote` WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `remoteTotal`,
(SELECT COUNT(*) FROM `peerCoordinate` WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `coordinateTotal`
(SELECT COUNT(*) FROM `peerRemote` WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `remoteTotal`
FROM `peer`
JOIN `peerSession` ON (`peerSession`.`peerId` = `peer`.`peerId`)
@ -795,8 +732,19 @@ class MySQL {
public function optimize() {
$this->_db->query('OPTIMIZE TABLE `peer`');
$this->_db->query('OPTIMIZE TABLE `peerSession`');
$this->_db->query('OPTIMIZE TABLE `peerConnection`');
$this->_db->query('OPTIMIZE TABLE `peerRemote`');
$this->_db->query('OPTIMIZE TABLE `peerCoordinate`');
$this->_db->query('OPTIMIZE TABLE `peerCoordinateRoute`');
$this->_db->query('OPTIMIZE TABLE `peerRemoteScheme`');
$this->_db->query('OPTIMIZE TABLE `peerRemoteHost`');
$this->_db->query('OPTIMIZE TABLE `peerRemotePort`');
$this->_db->query('OPTIMIZE TABLE `geo`');
$this->_db->query('OPTIMIZE TABLE `geoCountry`');
$this->_db->query('OPTIMIZE TABLE `geoCity`');
$this->_db->query('OPTIMIZE TABLE `geoCoordinate`');
}
}

4
src/library/sphinxql.php
View File

@ -66,11 +66,9 @@ class SphinxQL {
}
return sprintf(
'@peerAddress "%s" | @peerKey "%s" | @peerCoordinatePort "%s" | @peerCoordinateRoute "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s" | @geoCountryIsoCode "%s" | @geoCountryName "%s" | @geoCityName "%s"',
'@peerAddress "%s" | @peerKey "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s" | @geoCountryIsoCode "%s" | @geoCountryName "%s" | @geoCityName "%s"',
preg_replace('/[^A-z0-9\:\[\]]/', '', $peerAddress),
preg_replace('/[^A-z0-9]/', '', $keyword),
preg_replace('/[^\d]/', '', $keyword),
preg_replace('/[^\d]/', '', $keyword),
preg_replace('/[^A-z]/', '', $peerRemoteScheme),
preg_replace('/[^A-z0-9\.\:\[\]]/', '', $peerRemoteHost),
preg_replace('/[^\d]/', '', $peerRemotePort),

2
src/public/index.php
View File

@ -186,7 +186,7 @@ $peers = $memory->getByMethodCallback(
<sup class="label label-green font-size-12 font-width-normal cursor-default"><?php echo _('admin') ?></sup>
<?php } ?>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="" placeholder="<?php echo _('this, address, ip, geo, port, keyword...') ?>" />
<input type="text" name="query" value="" placeholder="<?php echo _('this, address, ip, geo, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>

14
src/public/peer.php
View File

@ -217,7 +217,7 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId);
<sup class="label label-green font-size-12 font-width-normal cursor-default"><?php echo _('admin') ?></sup>
<?php } ?>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="" placeholder="<?php echo _('this, address, ip, geo, port, keyword...') ?>" />
<input type="text" name="query" value="" placeholder="<?php echo _('this, address, ip, geo, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>
@ -263,12 +263,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId);
</sub>
</th>
<?php } ?>
<th class="text-center">
<?php echo _('Port') ?>
</th>
<th class="text-left">
<?php echo _('Coordinate') ?>
</th>
<th class="text-center">
<?php echo _('Geo') ?>
</th>
@ -286,8 +280,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId);
<?php echo $peerRemoteConnection->remote ?>
</td>
<?php } ?>
<td class="text-center"><?php echo $peerRemoteConnection->connectionPort ?></td>
<td class="text-left"><?php echo $peerRemoteConnection->route ?></td>
<td class="text-center">
<span class="cursor-default" title="<?php echo $peerRemoteConnection->geoCityName ?> <?php echo $peerRemoteConnection->geoCountryName ?>">
<?php echo $peerRemoteConnection->geoCountryIsoCode ?>
@ -377,10 +369,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId);
<td class="text-left"><?php echo _('Session') ?></td>
<td class="text-right"><?php echo number_format($peerInfo->sessionTotal) ?></td>
</tr>
<tr>
<td class="text-left"><?php echo _('Coordinate') ?></td>
<td class="text-right"><?php echo number_format($peerInfo->coordinateTotal) ?></td>
</tr>
</tbody>
</table>
</div>

30
src/public/search.php
View File

@ -89,7 +89,7 @@ $results = $sphinx->searchPeers($requestQuery,
<sup class="label label-green font-size-12 font-width-normal cursor-default"><?php echo _('admin') ?></sup>
<?php } ?>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="<?php echo !empty($_SERVER['REMOTE_ADDR']) && $requestQuery == $_SERVER['REMOTE_ADDR'] ? 'this' : htmlentities($requestQuery) ?>" placeholder="<?php echo _('this, address, ip, geo, port, keyword...') ?>" />
<input type="text" name="query" value="<?php echo !empty($_SERVER['REMOTE_ADDR']) && $requestQuery == $_SERVER['REMOTE_ADDR'] ? 'this' : htmlentities($requestQuery) ?>" placeholder="<?php echo _('this, address, ip, geo, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>
@ -112,8 +112,6 @@ $results = $sphinx->searchPeers($requestQuery,
<tr>
<th class="text-left"><?php echo _('Address') ?></th>
<th class="text-center"><?php echo _('Key') ?></th>
<th class="text-center"><?php echo _('Coordinate port') ?></th>
<th class="text-center"><?php echo _('Coordinate route') ?></th>
<th class="text-center"><?php echo _('Remote scheme') ?></th>
<th class="text-center"><?php echo _('Remote host') ?></th>
<th class="text-center"><?php echo _('Remote port') ?></th>
@ -140,28 +138,6 @@ $results = $sphinx->searchPeers($requestQuery,
</span>
<?php } ?>
</td>
<td class="text-center">
<?php if (false === strpos($result->peercoordinateport, $requestQuery)) { ?>
<span title="<?php echo $result->peercoordinateport ?>" class="font-size-22 cursor-default text-color-red">
&bull;
</span>
<?php } else { ?>
<span title="<?php echo $result->peercoordinateport ?>" class="font-size-22 cursor-default text-color-green">
&bull;
</span>
<?php } ?>
</td>
<td class="text-center">
<?php if (false === strpos($result->peercoordinateroute, $requestQuery)) { ?>
<span title="<?php echo $result->peercoordinateroute ?>" class="font-size-22 cursor-default text-color-red">
&bull;
</span>
<?php } else { ?>
<span title="<?php echo $result->peercoordinateroute ?>" class="font-size-22 cursor-default text-color-green">
&bull;
</span>
<?php } ?>
</td>
<td class="text-center">
<?php if (false === strpos(
$result->peerremotescheme,
@ -229,7 +205,7 @@ $results = $sphinx->searchPeers($requestQuery,
</tbody>
<tfoot>
<tr>
<td colspan="4" class="text-left">
<td colspan="6" class="text-left">
<?php if ($total >= WEBSITE_PEER_REMOTE_PAGINATION_LIMIT) { ?>
<?php if ($requestPage > 1) { ?>
<a href="search.php?query=<?php echo urlencode($requestQuery) ?>&page=<?php echo $requestPage - 1 ?>"><?php echo _('&larr;') ?></a>
@ -238,7 +214,7 @@ $results = $sphinx->searchPeers($requestQuery,
<a href="search.php?query=<?php echo urlencode($requestQuery) ?>&page=<?php echo $requestPage + 1 ?>"><?php echo _('&rarr;') ?></a>
<?php } ?>
</td>
<td colspan="5" class="text-right">
<td colspan="7" class="text-right">
<?php if (API_PEERS) { ?>
<div class="margin-top-8"><?php echo _('trackers') ?></div>
<?php foreach (API_PEERS as $tracker => $website) { ?>