diff --git a/composer.json b/composer.json index 876c015..d2212d8 100644 --- a/composer.json +++ b/composer.json @@ -4,7 +4,7 @@ "type": "project", "require": { "php": ">=8.1", - "yggverse/yggdrasilctl": ">=0.1.0", + "yggverse/yggdrasilctl": ">=1.0.0", "yggverse/parser": ">=0.1.0", "yggverse/graph": ">=0.1.0", "yggverse/cache": ">=0.3.0", diff --git a/database/yggstate.mwb b/database/yggstate.mwb index b0e06fb..04bb055 100644 Binary files a/database/yggstate.mwb and b/database/yggstate.mwb differ diff --git a/example/environment /sphinx.conf b/example/environment /sphinx.conf index 0b965cc..c2c2c7d 100644 --- a/example/environment /sphinx.conf +++ b/example/environment /sphinx.conf @@ -15,13 +15,6 @@ source peer : yggstate SELECT `peer`.`peerId`, \ `peer`.`address` AS `peerAddress`, \ `peer`.`key` AS `peerKey`, \ - (SELECT GROUP_CONCAT(DISTINCT `peerCoordinate`.`port`) \ - FROM `peerCoordinate` \ - WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `peerCoordinatePort`, \ - (SELECT GROUP_CONCAT(DISTINCT `peerCoordinateRoute`.`port`) \ - FROM `peerCoordinateRoute` \ - JOIN `peerCoordinate` ON (`peerCoordinate`.`peerCoordinateId` = `peerCoordinateRoute`.`peerCoordinateId`) \ - WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `peerCoordinateRoute`, \ (SELECT GROUP_CONCAT(DISTINCT `peerRemoteScheme`.`name`) \ FROM `peerRemote` \ JOIN `peerRemoteScheme` ON (`peerRemoteScheme`.`peerRemoteSchemeId` = `peerRemote`.`peerRemoteSchemeId`) \ @@ -56,8 +49,6 @@ source peer : yggstate sql_field_string = peerAddress sql_field_string = peerKey - sql_field_string = peerCoordinatePort - sql_field_string = peerCoordinateRoute sql_field_string = peerRemoteScheme sql_field_string = peerRemoteHost sql_field_string = peerRemotePort @@ -68,12 +59,12 @@ source peer : yggstate index peer { - source = peer - path = /var/lib/sphinxsearch/data/peer + source = peer + path = /var/lib/sphinxsearch/data/peer } indexer { - mem_limit = 256M - lemmatizer_cache = 256M + mem_limit = 256M + lemmatizer_cache = 256M } \ No newline at end of file diff --git a/src/config/app.php.example b/src/config/app.php.example index ac8acb0..e29f92a 100644 --- a/src/config/app.php.example +++ b/src/config/app.php.example @@ -75,11 +75,10 @@ define('WEBSITE_PEER_PORT_CHECK_TIMEOUT', 60 * 5); // API define('API_PEER_FIELDS', (array) [ + 'key', + 'address', 'bytes_recvd', 'bytes_sent', - 'coords', - 'key', - 'port', 'uptime', // ... ] diff --git a/src/crontab/crawler.php b/src/crontab/crawler.php index 2f108ea..54699d6 100644 --- a/src/crontab/crawler.php +++ b/src/crontab/crawler.php @@ -65,16 +65,6 @@ $debug = [ ] ] ], - 'coordinate' => [ - 'total' => [ - 'insert' => 0, - ], - 'route' => [ - 'total' => [ - 'insert' => 0, - ] - ] - ], 'connection' => [ 'total' => [ 'insert' => 0, @@ -134,8 +124,14 @@ try { @unlink(__DIR__ . '/../public/api/trackers.json'); // Update peers -if (!$connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) +$connectedPeersDebug = []; + +if (false === $connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers($connectedPeersDebug)) { + var_dump( + $connectedPeersDebug + ); + exit; } @@ -143,13 +139,13 @@ if (!$connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) /// Remove remote addresses $jsonPeers = []; -foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) +foreach ($connectedPeers as $connectedPeerInfo) { - foreach ($connectedPeerInfo as $connectedPeerInfoKey => $connectedPeerInfoValue) + foreach ($connectedPeerInfo as $key => $value) { - if (in_array($connectedPeerInfoKey, (array) API_PEER_FIELDS)) + if (in_array($key, (array) API_PEER_FIELDS)) { - $jsonPeers[$connectedPeerAddress][$connectedPeerInfoKey] = $connectedPeerInfoValue; + $jsonPeers[$connectedPeerInfo->address][$key] = $value; } } } @@ -171,20 +167,20 @@ if ($handle = fopen(__DIR__ . '/../public/api/trackers.json', 'w+')) // @TODO merge peers data from remote trackers // Collect connected peers -foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { +foreach ($connectedPeers as $connectedPeerInfo) { try { $db->beginTransaction(); // Init peer - if ($dbPeer = $db->findPeer($connectedPeerAddress)) { + if ($dbPeer = $db->findPeer($connectedPeerInfo->address)) { $dbPeerId = $dbPeer->peerId; } else { - if ($dbPeerId = $db->addPeer($connectedPeerAddress, $connectedPeerInfo->key, time())) { + if ($dbPeerId = $db->addPeer($connectedPeerInfo->address, $connectedPeerInfo->key, time())) { $debug['yggdrasil']['peer']['total']['insert']++; } @@ -229,51 +225,6 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { } } - // Init peer coordinate - if ($dbPeerCoordinate = $db->findLastPeerCoordinateByPeerId($dbPeerId)) { - - $dbPeerCoordinateId = $dbPeerCoordinate->peerCoordinateId; - - // Peer have changed it port, init new coordinate - if ($dbPeerCoordinate->port != $connectedPeerInfo->port) { - - if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { - - $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; - } - } - - } else { - - if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { - - $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; - } - } - - // Init peer coordinate route - $dbCoords = []; - foreach ($db->findPeerCoordinateRouteByCoordinateId($dbPeerCoordinateId) as $dbPeerCoordinateRoute) { - $dbCoords[$dbPeerCoordinateRoute->level] = $dbPeerCoordinateRoute->port; - } - - // Compare remote / local route, create new on changed - if ($dbCoords !== $connectedPeerInfo->coords) { - - if ($dbPeerCoordinateId = $db->addPeerCoordinate($dbPeerId, $connectedPeerInfo->port, time())) { - - $debug['yggdrasil']['peer']['coordinate']['total']['insert']++; - } - - foreach ($connectedPeerInfo->coords as $level => $port) { - - if ($db->addPeerCoordinateRoute($dbPeerCoordinateId, $level, $port)) { - - $debug['yggdrasil']['peer']['coordinate']['route']['total']['insert']++; - } - } - } - // Init peer remote if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) { @@ -375,7 +326,7 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { } else { if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude, - $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { + $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) { $debug['yggdrasil']['geo']['coordinate']['total']['insert']++; } @@ -428,9 +379,9 @@ foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) { } // Init peer connection - if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId)) { + if (!$db->findPeerConnection($dbPeerSessionId, $dbPeerRemoteId)) { - if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, $dbPeerCoordinateId, time())) { + if ($db->addPeerConnection($dbPeerSessionId, $dbPeerRemoteId, time())) { $debug['yggdrasil']['peer']['connection']['total']['insert']++; } diff --git a/src/library/mysql.php b/src/library/mysql.php index 5e6c0dd..5356d83 100644 --- a/src/library/mysql.php +++ b/src/library/mysql.php @@ -457,84 +457,32 @@ class MySQL { } // Peer connection - public function addPeerConnection(int $peerSessionId, int $peerRemoteId, int $peerCoordinateId, int $timeAdded) { + public function addPeerConnection(int $peerSessionId, int $peerRemoteId, int $timeAdded) { $this->_debug->query->insert->total++; $query = $this->_db->prepare('INSERT INTO `peerConnection` SET `peerSessionId` = ?, `peerRemoteId` = ?, - `peerCoordinateId` = ?, `timeAdded` = ?'); - $query->execute([$peerSessionId, $peerRemoteId, $peerCoordinateId, $timeAdded]); + $query->execute([$peerSessionId, $peerRemoteId, $timeAdded]); return $this->_db->lastInsertId(); } - public function findPeerConnection(int $peerSessionId, int $peerRemoteId, int $peerCoordinateId) { + public function findPeerConnection(int $peerSessionId, int $peerRemoteId) { $this->_debug->query->select->total++; $query = $this->_db->prepare('SELECT * FROM `peerConnection` - WHERE `peerSessionId` = ? AND `peerRemoteId` = ? AND `peerCoordinateId` = ? + WHERE `peerSessionId` = ? AND `peerRemoteId` = ? LIMIT 1'); - $query->execute([$peerSessionId, $peerRemoteId, $peerCoordinateId]); + $query->execute([$peerSessionId, $peerRemoteId]); return $query->fetch(); } - // Peer coordinate - // https://github.com/matrix-org/pinecone/wiki/2.-Spanning-Tree#coordinates - - public function addPeerCoordinate(int $peerId, int $port, int $timeAdded) { - - $this->_debug->query->insert->total++; - - $query = $this->_db->prepare('INSERT INTO `peerCoordinate` SET `peerId` = ?, - `port` = ?, - `timeAdded` = ?'); - - $query->execute([$peerId, $port, $timeAdded]); - - return $this->_db->lastInsertId(); - } - - public function addPeerCoordinateRoute(int $peerCoordinateId, int $level, int $port) { - - $this->_debug->query->insert->total++; - - $query = $this->_db->prepare('INSERT INTO `peerCoordinateRoute` SET `peerCoordinateId` = ?, - `level` = ?, - `port` = ?'); - - $query->execute([$peerCoordinateId, $level, $port]); - - return $this->_db->lastInsertId(); - } - - public function findLastPeerCoordinateByPeerId(int $peerId) { - - $this->_debug->query->select->total++; - - $query = $this->_db->prepare('SELECT * FROM `peerCoordinate` WHERE `peerId` = ? ORDER BY `peerCoordinateId` DESC LIMIT 1'); - - $query->execute([$peerId]); - - return $query->fetch(); - } - - public function findPeerCoordinateRouteByCoordinateId(int $peerCoordinateId) { - - $this->_debug->query->select->total++; - - $query = $this->_db->prepare('SELECT * FROM `peerCoordinateRoute` WHERE `peerCoordinateId` = ? ORDER BY `level` ASC'); - - $query->execute([$peerCoordinateId]); - - return $query->fetchAll(); - } - // Analytics public function getPeerFirstByTimeAdded() { @@ -703,15 +651,6 @@ class MySQL { `peerRemoteHost`.`name` AS `remoteHost`, `peerRemotePort`.`name` AS `remotePort`, - `peerCoordinate`.`port` AS `connectionPort`, - - ( - SELECT GROUP_CONCAT(`port` SEPARATOR ' → ') - FROM `peerCoordinateRoute` - WHERE `peerCoordinateRoute`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId` - ORDER BY `peerCoordinateRoute`.`level` ASC - ) AS `route`, - CONCAT ( IF (`peerRemotePort`.`name` IS NOT NULL, @@ -728,7 +667,6 @@ class MySQL { JOIN `peerSession` ON (`peerSession`.`peerSessionId` = `peerConnection`.`peerSessionId`) JOIN `peerRemote` ON (`peerRemote`.`peerRemoteId` = `peerConnection`.`peerRemoteId`) - JOIN `peerCoordinate` ON (`peerCoordinate`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId`) JOIN `peerRemoteScheme` ON (`peerRemoteScheme`.`peerRemoteSchemeId` = `peerRemote`.`peerRemoteSchemeId`) JOIN `peerRemoteHost` ON (`peerRemoteHost`.`peerRemoteHostId` = `peerRemote`.`peerRemoteHostId`) @@ -774,8 +712,7 @@ class MySQL { ROUND(AVG(`peerSession`.`uptime`)) AS `uptimeAvg`, (SELECT COUNT(*) FROM `peerSession` WHERE `peerSession`.`peerId` = `peer`.`peerId`) AS `sessionTotal`, - (SELECT COUNT(*) FROM `peerRemote` WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `remoteTotal`, - (SELECT COUNT(*) FROM `peerCoordinate` WHERE `peerCoordinate`.`peerId` = `peer`.`peerId`) AS `coordinateTotal` + (SELECT COUNT(*) FROM `peerRemote` WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `remoteTotal` FROM `peer` JOIN `peerSession` ON (`peerSession`.`peerId` = `peer`.`peerId`) @@ -795,8 +732,19 @@ class MySQL { public function optimize() { $this->_db->query('OPTIMIZE TABLE `peer`'); + + $this->_db->query('OPTIMIZE TABLE `peerSession`'); + + $this->_db->query('OPTIMIZE TABLE `peerConnection`'); + $this->_db->query('OPTIMIZE TABLE `peerRemote`'); - $this->_db->query('OPTIMIZE TABLE `peerCoordinate`'); - $this->_db->query('OPTIMIZE TABLE `peerCoordinateRoute`'); + $this->_db->query('OPTIMIZE TABLE `peerRemoteScheme`'); + $this->_db->query('OPTIMIZE TABLE `peerRemoteHost`'); + $this->_db->query('OPTIMIZE TABLE `peerRemotePort`'); + + $this->_db->query('OPTIMIZE TABLE `geo`'); + $this->_db->query('OPTIMIZE TABLE `geoCountry`'); + $this->_db->query('OPTIMIZE TABLE `geoCity`'); + $this->_db->query('OPTIMIZE TABLE `geoCoordinate`'); } } diff --git a/src/library/sphinxql.php b/src/library/sphinxql.php index 154880e..b1d16a7 100644 --- a/src/library/sphinxql.php +++ b/src/library/sphinxql.php @@ -66,11 +66,9 @@ class SphinxQL { } return sprintf( - '@peerAddress "%s" | @peerKey "%s" | @peerCoordinatePort "%s" | @peerCoordinateRoute "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s" | @geoCountryIsoCode "%s" | @geoCountryName "%s" | @geoCityName "%s"', + '@peerAddress "%s" | @peerKey "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s" | @geoCountryIsoCode "%s" | @geoCountryName "%s" | @geoCityName "%s"', preg_replace('/[^A-z0-9\:\[\]]/', '', $peerAddress), preg_replace('/[^A-z0-9]/', '', $keyword), - preg_replace('/[^\d]/', '', $keyword), - preg_replace('/[^\d]/', '', $keyword), preg_replace('/[^A-z]/', '', $peerRemoteScheme), preg_replace('/[^A-z0-9\.\:\[\]]/', '', $peerRemoteHost), preg_replace('/[^\d]/', '', $peerRemotePort), diff --git a/src/public/index.php b/src/public/index.php index aaad723..cc5cfe7 100644 --- a/src/public/index.php +++ b/src/public/index.php @@ -186,7 +186,7 @@ $peers = $memory->getByMethodCallback(
- +
diff --git a/src/public/peer.php b/src/public/peer.php index 790c107..7db3c3e 100644 --- a/src/public/peer.php +++ b/src/public/peer.php @@ -217,7 +217,7 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId);
- +
@@ -263,12 +263,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId); - - - - - - @@ -286,8 +280,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId); remote ?> - connectionPort ?> - route ?> geoCountryIsoCode ?> @@ -377,10 +369,6 @@ $peerPortStatuses = $db->findLastPeerPortStatusesByPeerId($requestPeerId); sessionTotal) ?> - - - coordinateTotal) ?> - diff --git a/src/public/search.php b/src/public/search.php index da942c6..9e74601 100644 --- a/src/public/search.php +++ b/src/public/search.php @@ -89,7 +89,7 @@ $results = $sphinx->searchPeers($requestQuery,
- +
@@ -112,8 +112,6 @@ $results = $sphinx->searchPeers($requestQuery, - - @@ -140,28 +138,6 @@ $results = $sphinx->searchPeers($requestQuery,
- - peercoordinateport, $requestQuery)) { ?> - - • - - - - • - - - - - peercoordinateroute, $requestQuery)) { ?> - - • - - - - • - - - peerremotescheme, @@ -229,7 +205,7 @@ $results = $sphinx->searchPeers($requestQuery, - + = WEBSITE_PEER_REMOTE_PAGINATION_LIMIT) { ?> 1) { ?> @@ -238,7 +214,7 @@ $results = $sphinx->searchPeers($requestQuery, - +
$website) { ?>