YGGverse/YGGstate
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGstate.git synced 2025-01-24 13:34:27 +00:00
Code Issues Projects Releases Wiki Activity

add geo-IP/GeoLite2 integration

Browse Source
This commit is contained in:
ghost 2023-08-14 19:29:10 +03:00
parent baed74f0a8
commit 8a4afa6e91
12 changed files with 323 additions and 24 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

23
example/environment /sphinx.conf
View File

@ -33,7 +33,25 @@ source peer : yggstate
(SELECT GROUP_CONCAT(DISTINCT `peerRemotePort`.`name`) \
FROM `peerRemote` \
JOIN `peerRemotePort` ON (`peerRemotePort`.`peerRemotePortId` = `peerRemote`.`peerRemotePortId`) \
WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `peerRemotePort` \
WHERE `peerRemote`.`peerId` = `peer`.`peerId`) AS `peerRemotePort`, \
(SELECT GROUP_CONCAT(DISTINCT `geoCountry`.`isoCode`) \
FROM `peerRemote` \
JOIN `geo` ON (`geo`.`geoId` = `peerRemote`.`geoId`) \
JOIN `geoCountry` ON (`geoCountry`.`geoCountryId` = `geo`.`geoCountryId`) \
WHERE `peerRemote`.`peerId` = `peer`.`peerId` \
) AS `geoCountryIsoCode`, \
(SELECT GROUP_CONCAT(DISTINCT `geoCountry`.`name`) \
FROM `peerRemote` \
JOIN `geo` ON (`geo`.`geoId` = `peerRemote`.`geoId`) \
JOIN `geoCountry` ON (`geoCountry`.`geoCountryId` = `geo`.`geoCountryId`) \
WHERE `peerRemote`.`peerId` = `peer`.`peerId` \
) AS `geoCountryName`, \
(SELECT GROUP_CONCAT(DISTINCT `geoCity`.`name`) \
FROM `peerRemote` \
JOIN `geo` ON (`geo`.`geoId` = `peerRemote`.`geoId`) \
JOIN `geoCity` ON (`geoCity`.`geoCityId` = `geo`.`geoCityId`) \
WHERE `peerRemote`.`peerId` = `peer`.`peerId` \
) AS `geoCityName` \
FROM `peer`\
sql_field_string = peerAddress
@ -43,6 +61,9 @@ source peer : yggstate
sql_field_string = peerRemoteScheme
sql_field_string = peerRemoteHost
sql_field_string = peerRemotePort
sql_field_string = geoCountryIsoCode
sql_field_string = geoCountryName
sql_field_string = geoCityName
}
index peer

1
.gitignore vendored
View File

@ -2,6 +2,7 @@
/vendor/
/database/yggstate.mwb.bak
/storage/GeoLite2/*
/src/config/app.php

3
composer.json
View File

@ -7,7 +7,8 @@
"yggverse/yggdrasilctl": ">=0.1.0",
"yggverse/parser": ">=0.1.0",
"yggverse/graph": ">=0.1.0",
"yggverse/cache": ">=0.3.0"
"yggverse/cache": ">=0.3.0",
"geoip2/geoip2": "~2.0"
},
"license": "MIT",
"autoload": {

BIN
database/yggstate.mwb
View File

Binary file not shown.

BIN
media/db-prototype.png
View File

Binary file not shown.
Side by Side Swipe Overlay

Before

Width:  |  Height:  |  Size: 116 KiB

After

Width:  |  Height:  |  Size: 154 KiB

4
src/config/app.php.example
View File

@ -58,6 +58,10 @@ define('MEMCACHED_HOST', 'localhost');
define('MEMCACHED_NAMESPACE', 'yggstate');
define('MEMCACHED_TIMEOUT', 60 * 5);
// GeoIp2 integration by https://www.maxmind.com
define('GEOIP_LITE_2_COUNTRY_DB', __DIR__ . '/../../storage/GeoLite2/GeoLite2-Country.mmdb');
define('GEOIP_LITE_2_CITY_DB', __DIR__ . '/../../storage/GeoLite2/GeoLite2-City.mmdb');
// Webapp
define('WEBSITE_URL', '');
define('WEBSITE_NAME', 'YGGstate');

124
src/crontab/crawler.php
View File

@ -80,11 +80,31 @@ $debug = [
'insert' => 0,
],
],
],
'geo' => [
'total' => [
'insert' => 0,
],
'country' => [
'total' => [
'insert' => 0,
],
],
'city' => [
'total' => [
'insert' => 0,
],
],
'coordinate' => [
'total' => [
'insert' => 0,
],
],
]
]
];
// Connect database
// Connect DB
try {
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
@ -96,8 +116,21 @@ try {
exit;
}
// GeoIp2
try {
$geoIp2Country = new GeoIp2\Database\Reader(GEOIP_LITE_2_COUNTRY_DB);
$geoIp2City = new GeoIp2\Database\Reader(GEOIP_LITE_2_CITY_DB);
} catch(Exception $e) {
var_dump($e);
exit;
}
// Collect connected peers
if ($connectedPeers = \Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
if ($connectedPeers = Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
foreach ($connectedPeers as $connectedPeerAddress => $connectedPeerInfo) {
@ -203,7 +236,7 @@ if ($connectedPeers = \Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
}
// Init peer remote
if ($connectedPeerRemoteUrl = \Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
if ($connectedPeerRemoteUrl = Yggverse\Parser\Url::parse($connectedPeerInfo->remote)) {
// Init peer scheme
if ($dbPeerRemoteScheme = $db->findPeerRemoteScheme($connectedPeerRemoteUrl->host->scheme)) {
@ -244,8 +277,90 @@ if ($connectedPeers = \Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
}
}
// Init geo data
/// Country
$countryIsoCode = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->isoCode;
$countryName = $geoIp2Country->country($connectedPeerRemoteUrl->host->name)->country->name;
$dbGeoCountryId = null;
if (!empty($countryIsoCode) && !empty($countryName)) {
if ($dbGeoCountry = $db->findGeoCountryByIsoCode($countryIsoCode)) {
$dbGeoCountryId = $dbGeoCountry->geoCountryId;
} else {
if ($dbGeoCountryId = $db->addGeoCountry($countryIsoCode, $countryName)) {
$debug['yggdrasil']['geo']['country']['total']['insert']++;
}
}
}
/// City
$cityName = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->city->name;
$dbGeoCityId = null;
if (!empty($cityName)) {
if ($dbGeoCity = $db->findGeoCityByName($cityName)) {
$dbGeoCityId = $dbGeoCity->geoCityId;
} else {
if ($dbGeoCityId = $db->addGeoCity($cityName)) {
$debug['yggdrasil']['geo']['city']['total']['insert']++;
}
}
}
/// Coordinate
$latitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude;
$longitude = $geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude;
$dbGeoCoordinateId = null;
if (!empty($latitude) && !empty($longitude)) {
if ($dbGeoCoordinate = $db->findGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$dbGeoCoordinateId = $dbGeoCoordinate->geoCoordinateId;
} else {
if ($dbGeoCoordinateId = $db->addGeoCoordinate($geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->latitude,
$geoIp2City->city($connectedPeerRemoteUrl->host->name)->location->longitude)) {
$debug['yggdrasil']['geo']['coordinate']['total']['insert']++;
}
}
}
/// Geo
$dbGeoId = null;
if ($dbGeo = $db->findGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$dbGeoId = $dbGeo->geoId;
} else {
if ($dbGeoId = $db->addGeo($dbGeoCountryId, $dbGeoCityId, $dbGeoCoordinateId)) {
$debug['yggdrasil']['geo']['total']['insert']++;
}
}
// Init peer remote
if ($dbPeerRemote = $db->findPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId)) {
@ -255,6 +370,7 @@ if ($connectedPeers = \Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
} else {
if ($dbPeerRemoteId = $db->addPeerRemote($dbPeerId,
$dbGeoId,
$dbPeerRemoteSchemeId,
$dbPeerRemoteHostId,
$dbPeerRemotePortId,
@ -267,6 +383,8 @@ if ($connectedPeers = \Yggverse\Yggdrasilctl\Yggdrasil::getPeers()) {
// If something went wrong with URL parse, skip next operations for this peer
} else {
$db->rollBack();
continue;
}

144
src/library/mysql.php
View File

@ -114,29 +114,127 @@ class MySQL {
return $query->fetchAll();
}
// Geo
public function findGeo(mixed $geoCountryId, mixed $geoCityId, mixed $geoCoordinateId) { // int|null
$this->_debug->query->select->total++;
// @TODO
// UNIQUE keys does not applying for the NULL values,
// another problem that MySQL return no results for = NULL condition
// if someone have better solution, feel free to PR this issue https://github.com/YGGverse/YGGstate/pulls
$query = $this->_db->query("SELECT * FROM `geo`
WHERE `geoCountryId` " . (empty($geoCountryId) ? " IS NULL " : " = " . (int) $geoCountryId ) . "
AND `geoCityId` " . (empty($geoCityId) ? " IS NULL " : " = " . (int) $geoCityId ) . "
AND `geoCoordinateId` " . (empty($geoCoordinateId) ? " IS NULL " : " = " . (int) $geoCoordinateId) . "
LIMIT 1");
return $query->fetch();
}
public function addGeo(mixed $geoCountryId, mixed $geoCityId, mixed $geoCoordinateId) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `geo` SET `geoCountryId` = ?,
`geoCityId` = ?,
`geoCoordinateId` = ?');
$query->execute([$geoCountryId, $geoCityId, $geoCoordinateId]);
return $this->_db->lastInsertId();
}
public function findGeoCountryByIsoCode(string $isoCode) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `geoCountry` WHERE `isoCode` = ? LIMIT 1');
$query->execute([$isoCode]);
return $query->fetch();
}
public function addGeoCountry(string $isoCode, string $name) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `geoCountry` SET `isoCode` = ?, `name` = ?');
$query->execute([$isoCode, $name]);
return $this->_db->lastInsertId();
}
public function findGeoCityByName(string $name) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `geoCity` WHERE `name` = ? LIMIT 1');
$query->execute([$name]);
return $query->fetch();
}
public function addGeoCity(string $name) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `geoCity` SET `name` = ?');
$query->execute([$name]);
return $this->_db->lastInsertId();
}
public function findGeoCoordinate(float $latitude, float $longitude) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `geoCoordinate` WHERE `point` = POINT(?, ?) LIMIT 1');
$query->execute([$latitude, $longitude]);
return $query->fetch();
}
public function addGeoCoordinate(float $latitude, float $longitude) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `geoCoordinate` SET `point` = POINT(?, ?)');
$query->execute([$latitude, $longitude]);
return $this->_db->lastInsertId();
}
// Peer remote
public function addPeerRemote(int $peerId, int $peerRemoteSchemeId, int $peerRemoteHostId, int $peerRemotePortId, int $timeAdded) {
public function addPeerRemote(int $peerId, int $geoId, int $peerRemoteSchemeId, int $peerRemoteHostId, int $peerRemotePortId, int $timeAdded) {
$this->_debug->query->insert->total++;
$query = $this->_db->prepare('INSERT INTO `peerRemote` SET `peerId` = ?,
`geoId` = ?,
`peerRemoteSchemeId` = ?,
`peerRemoteHostId` = ?,
`peerRemotePortId` = ?,
`timeAdded` = ?');
$query->execute([$peerId, $peerRemoteSchemeId, $peerRemoteHostId, $peerRemotePortId, $timeAdded]);
$query->execute([$peerId, $geoId, $peerRemoteSchemeId, $peerRemoteHostId, $peerRemotePortId, $timeAdded]);
return $this->_db->lastInsertId();
}
public function findPeerRemote(int $peerId, int $peerRemoteSchemeId, int $peerRemoteHostId, int $peerRemotePortId) {
public function findPeerRemote(int $peerId, int $geoId, int $peerRemoteSchemeId, int $peerRemoteHostId, int $peerRemotePortId) {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `peerRemote` WHERE `peerId` = ? AND `peerRemoteSchemeId` = ? AND `peerRemoteHostId` = ? AND `peerRemotePortId` = ? LIMIT 1');
$query = $this->_db->prepare('SELECT * FROM `peerRemote` WHERE `peerId` = ? AND `geoId` = ? AND `peerRemoteSchemeId` = ? AND `peerRemoteHostId` = ? AND `peerRemotePortId` = ? LIMIT 1');
$query->execute([$peerId, $peerRemoteSchemeId, $peerRemoteHostId, $peerRemotePortId]);
$query->execute([$peerId, $geoId, $peerRemoteSchemeId, $peerRemoteHostId, $peerRemotePortId]);
return $query->fetch();
}
@ -189,9 +287,21 @@ class MySQL {
$this->_debug->query->select->total++;
$query = $this->_db->prepare('SELECT * FROM `peerRemotePort` WHERE `name` = ? LIMIT 1');
// @TODO
// UNIQUE keys does not applying for the NULL values,
// another problem that MySQL return no results for = NULL condition
// if someone have better solution, feel free to PR this issue https://github.com/YGGverse/YGGstate/pulls
if (empty($name)) {
$query = $this->_db->query('SELECT * FROM `peerRemotePort` WHERE `name` IS NULL LIMIT 1');
} else {
$query = $this->_db->prepare('SELECT * FROM `peerRemotePort` WHERE `name` = ? LIMIT 1');
$query->execute([$name]);
}
$query->execute([$name]);
return $query->fetch();
}
@ -517,6 +627,13 @@ class MySQL {
`peerCoordinate`.`port` AS `connectionPort`,
(
SELECT GROUP_CONCAT(`port` SEPARATOR ' → ')
FROM `peerCoordinateRoute`
WHERE `peerCoordinateRoute`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId`
ORDER BY `peerCoordinateRoute`.`level` ASC
) AS `route`,
CONCAT
(
IF (`peerRemotePort`.`name` IS NOT NULL,
@ -525,12 +642,9 @@ class MySQL {
)
) AS `remote`,
(
SELECT GROUP_CONCAT(`port` SEPARATOR ' → ')
FROM `peerCoordinateRoute`
WHERE `peerCoordinateRoute`.`peerCoordinateId` = `peerConnection`.`peerCoordinateId`
ORDER BY `peerCoordinateRoute`.`level` ASC
) AS `route`
`geoCountry`.`isoCode` AS `geoCountryIsoCode`,
`geoCountry`.`name` `geoCountryName`,
`geoCity`.`name` AS `geoCityName`
FROM `peerConnection`
@ -544,6 +658,10 @@ class MySQL {
JOIN `peer` ON (`peer`.`peerId` = `peerRemote`.`peerId`)
LEFT JOIN `geo` ON (`geo`.`geoId` = `peerRemote`.`geoId`)
LEFT JOIN `geoCountry` ON (`geoCountry`.`geoCountryId` = `geo`.`geoCountryId`)
LEFT JOIN `geoCity` ON (`geoCity`.`geoCityId` = `geo`.`geoCityId`)
WHERE `peerRemote`.`peerId` = :peerId
GROUP BY `peerConnection`.`peerConnectionId`

5
src/library/sphinxql.php
View File

@ -66,7 +66,7 @@ class SphinxQL {
}
return sprintf(
'@peerAddress "%s" | @peerKey "%s" | @peerCoordinatePort "%s" | @peerCoordinateRoute "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s"',
'@peerAddress "%s" | @peerKey "%s" | @peerCoordinatePort "%s" | @peerCoordinateRoute "%s" | @peerRemoteScheme "%s" | @peerRemoteHost "%s" | @peerRemotePort "%s" | @geoCountryIsoCode "%s" | @geoCountryName "%s" | @geoCityName "%s"',
preg_replace('/[^A-z0-9\:\[\]]/', '', $peerAddress),
preg_replace('/[^A-z0-9]/', '', $keyword),
preg_replace('/[^\d]/', '', $keyword),
@ -74,6 +74,9 @@ class SphinxQL {
preg_replace('/[^A-z]/', '', $peerRemoteScheme),
preg_replace('/[^A-z0-9\.\:\[\]]/', '', $peerRemoteHost),
preg_replace('/[^\d]/', '', $peerRemotePort),
preg_replace('/[^A-z]/', '', $keyword),
preg_replace('/[^A-z]/', '', $keyword),
preg_replace('/[^A-z]/', '', $keyword),
);
}
}

2
src/public/index.php
View File

@ -181,7 +181,7 @@ $peers = $memory->getByMethodCallback(
<div class="row">
<a class="logo" href="<?php echo WEBSITE_URL ?>"><?php echo str_replace('YGG', '<span>YGG</span>', WEBSITE_NAME) ?></a>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="" placeholder="<?php echo _('address, ip, port, keyword...') ?>" />
<input type="text" name="query" value="" placeholder="<?php echo _('address, ip, geo, port, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>

12
src/public/peer.php
View File

@ -135,7 +135,7 @@ $peerInfo = $memory->getByMethodCallback($db, 'getPeerInfo', [$requestPeerId]);
<div class="row">
<a class="logo" href="<?php echo WEBSITE_URL ?>"><?php echo str_replace('YGG', '<span>YGG</span>', WEBSITE_NAME) ?></a>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="" placeholder="<?php echo _('address, ip, port, keyword...') ?>" />
<input type="text" name="query" value="" placeholder="<?php echo _('address, ip, geo, port, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>
@ -173,6 +173,9 @@ $peerInfo = $memory->getByMethodCallback($db, 'getPeerInfo', [$requestPeerId]);
<th class="text-left">
<?php echo _('Coordinate') ?>
</th>
<th class="text-center">
<?php echo _('Geo') ?>
</th>
<th class="text-center">
<?php echo _('Online') ?>
</th>
@ -185,6 +188,11 @@ $peerInfo = $memory->getByMethodCallback($db, 'getPeerInfo', [$requestPeerId]);
<td class="text-left"><?php echo $peerRemoteConnection->remote ?></td>
<td class="text-center"><?php echo $peerRemoteConnection->connectionPort ?></td>
<td class="text-left"><?php echo $peerRemoteConnection->route ?></td>
<td class="text-center">
<span class="cursor-default" title="<?php echo $peerRemoteConnection->geoCityName ?> <?php echo $peerRemoteConnection->geoCountryName ?>">
<?php echo $peerRemoteConnection->geoCountryIsoCode ?>
<span>
</td>
<td class="text-center">
<span class="font-size-22 cursor-default <?php echo $i == 0 && $peerRemoteConnection->timeOnline > time() - WEBSITE_PEER_REMOTE_TIME_ONLINE_TIMEOUT ? 'text-color-green' : 'text-color-red' ?>">
&bull;
@ -196,7 +204,7 @@ $peerInfo = $memory->getByMethodCallback($db, 'getPeerInfo', [$requestPeerId]);
<?php if ($i >= WEBSITE_PEER_REMOTE_PAGINATION_LIMIT) { ?>
<tfoot>
<tr>
<td colspan="5" class="text-left">
<td colspan="6" class="text-left">
<?php if ($requestPage > 1) { ?>
<a href="<?php echo WEBSITE_URL ?>/peer.php?peerId=<?php echo $requestPeerId ?>&sort=<?php echo $requestSort ?>&page=<?php echo $requestPage - 1 ?>"><?php echo _('&larr;') ?></a>
<?php } ?>

29
src/public/search.php
View File

@ -74,7 +74,7 @@ $results = $sphinx->searchPeers($requestQuery,
<div class="row">
<a class="logo" href="<?php echo WEBSITE_URL ?>"><?php echo str_replace('YGG', '<span>YGG</span>', WEBSITE_NAME) ?></a>
<form name="search" method="get" action="<?php echo WEBSITE_URL ?>/search.php">
<input type="text" name="query" value="<?php echo htmlentities($requestQuery) ?>" placeholder="<?php echo _('address, ip, port, keyword...') ?>" />
<input type="text" name="query" value="<?php echo htmlentities($requestQuery) ?>" placeholder="<?php echo _('address, ip, geo, port, keyword...') ?>" />
<button type="submit"><?php echo _('search') ?></button>
</form>
</div>
@ -102,6 +102,8 @@ $results = $sphinx->searchPeers($requestQuery,
<th class="text-center"><?php echo _('Remote scheme') ?></th>
<th class="text-center"><?php echo _('Remote host') ?></th>
<th class="text-center"><?php echo _('Remote port') ?></th>
<th class="text-center"><?php echo _('Country') ?></th>
<th class="text-center"><?php echo _('City') ?></th>
</tr>
</thead>
<tbody>
@ -184,13 +186,36 @@ $results = $sphinx->searchPeers($requestQuery,
</span>
<?php } ?>
</td>
<td class="text-center">
<?php if (false === stripos($result->geocountryname, $requestQuery) &&
false === stripos($result->geocountryisocode, $requestQuery)) { ?>
<span title="<?php echo $result->geocountryname ?> <?php echo $result->geocountryisocode ?>" class="font-size-22 cursor-default text-color-red">
&bull;
</span>
<?php } else { ?>
<span title="<?php echo $result->geocountryname ?> <?php echo $result->geocountryisocode ?>" class="font-size-22 cursor-default text-color-green">
&bull;
</span>
<?php } ?>
</td>
<td class="text-center">
<?php if (false === stripos($result->geocityname, $requestQuery)) { ?>
<span title="<?php echo $result->geocityname ?>" class="font-size-22 cursor-default text-color-red">
&bull;
</span>
<?php } else { ?>
<span title="<?php echo $result->geocityname ?>" class="font-size-22 cursor-default text-color-green">
&bull;
</span>
<?php } ?>
</td>
</tr>
<?php } ?>
</tbody>
<?php if ($total >= WEBSITE_PEER_REMOTE_PAGINATION_LIMIT) { ?>
<tfoot>
<tr>
<td colspan="5" class="text-left">
<td colspan="9" class="text-left">
<?php if ($requestPage > 1) { ?>
<a href="search.php?query=<?php echo urlencode($requestQuery) ?>&page=<?php echo $requestPage - 1 ?>"><?php echo _('&larr;') ?></a>
<?php } ?>