twisterarmy/twister-analytics-crawler
1
0
Fork 0
mirror of https://github.com/twisterarmy/twister-analytics-crawler.git synced 2025-01-30 00:14:15 +00:00
Code Issues Projects Releases Wiki Activity

initial commit

Browse Source
This commit is contained in:
D4708 2021-05-30 01:33:36 +03:00
parent c12405280e
commit e17b16030d
10 changed files with 586 additions and 0 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

35
config-default.php Normal file
View File

@ -0,0 +1,35 @@
<?php
// Debug
ini_set('display_errors', '1');
ini_set('display_startup_errors', '1');
error_reporting(E_ALL);
// Twister
define('TWISTER_PROTOCOL', 'http');
define('TWISTER_HOST', 'localhost');
define('TWISTER_PORT', 28332);
define('TWISTER_USERNAME', '');
define('TWISTER_PASSWORD', '');
// Geoplugin
define('GEOPLUGIN_PROTOCOL', 'http');
define('GEOPLUGIN_HOST', 'www.geoplugin.net');
define('GEOPLUGIN_PORT', 80);
// Torproject
define('TORPROJECT_PROTOCOL', 'https');
define('TORPROJECT_HOST', 'check.torproject.org');
define('TORPROJECT_PORT', 443);
// DB
define('DB_HOSTNAME', 'localhost');
define('DB_PORT', '3306');
define('DB_DATABASE', '');
define('DB_USERNAME', '');
define('DB_PASSWORD', '');
// Options
define('EMAIL_ONLINE_PEERS', false); // email address|false
define('EMAIL_OFFLINE_PEERS', false); // email address|false
define('EMAIL_NEW_PEERS', false); // email address|false

95
curl/curl.php Normal file
View File

@ -0,0 +1,95 @@
<?php
class Curl {
private $_curl;
private $_protocol;
private $_host;
private $_port;
public function __construct($protocol, $host, $port, $username = false, $password = false) {
$this->_protocol = $protocol;
$this->_host = $host;
$this->_port = $port;
$this->_curl = curl_init();
if ($username && $password) {
$headers = [
'Content-Type: application/json',
sprintf('Authorization: Basic %s', base64_encode($username . ':' . $password)),
];
} else {
$headers = [
'Content-Type: application/json',
];
}
curl_setopt_array($this->_curl, [CURLOPT_RETURNTRANSFER => true,
CURLOPT_FOLLOWLOCATION => true,
CURLOPT_FRESH_CONNECT => true,
//CURLOPT_VERBOSE => true,
CURLOPT_HTTPHEADER => $headers,
]);
}
public function __destruct() {
curl_close($this->_curl);
}
public function sanitize($string, $lowercase = false) {
// Lowercase
if ($lowercase) {
$string = mb_strtolower($string, 'UTF-8');
}
return $string;
}
protected function prepare($uri, $method, $timeout = 30, array $postfields = [], $verify_ssl = true, $verify_host = true) {
curl_setopt($this->_curl, CURLOPT_URL, $this->_protocol . '://' . $this->_host . ':' . $this->_port . '/' . $uri);
curl_setopt($this->_curl, CURLOPT_CONNECTTIMEOUT, $timeout);
curl_setopt($this->_curl, CURLOPT_TIMEOUT, $timeout);
if ($verify_ssl === false) {
curl_setopt($this->_curl, CURLOPT_SSL_VERIFYPEER, false);
}
if ($verify_host === false) {
curl_setopt($this->_curl, CURLOPT_SSL_VERIFYHOST, false);
}
if ($method) {
curl_setopt($this->_curl, CURLOPT_CUSTOMREQUEST, $method);
}
if ($method == 'POST' && $postfields) {
curl_setopt($this->_curl, CURLOPT_POSTFIELDS, json_encode($postfields));
}
}
protected function execute($json = true) {
$response = curl_exec($this->_curl);
$errorNumber = curl_errno($this->_curl);
$errorText = curl_error($this->_curl);
if ($errorNumber > 0) {
return false;
}
if ($response) {
if ($json) {
return json_decode($response, true);
} else {
return $response;
}
}
return false;
}
}

25
curl/geoplugin.php Normal file
View File

@ -0,0 +1,25 @@
<?php
class CurlGeoPlugin extends Curl {
public function getLocation($ip) {
$this->prepare('json.gp?ip=' . $ip, 'GET');
if ($response = $this->execute()) {
switch (false) {
case isset($response['geoplugin_city']):
case isset($response['geoplugin_countryCode']):
case isset($response['geoplugin_latitude']):
case isset($response['geoplugin_longitude']):
return false;
}
return $response;
}
return false;
}
}

31
curl/peer.php Normal file
View File

@ -0,0 +1,31 @@
<?php
class CurlPeer extends Curl {
public function getAll() {
$this->prepare('', 'POST', 30, ['jsonrpc' => '2.0',
'method' => 'getpeerinfo',
'params' => [],
'id' => 1], false, false);
if ($response = $this->execute()) {
if (isset($response['result'])) {
$peers = [];
foreach ($response['result'] as $peer) {
# @TODO validate
if (isset($peer['addr'])) {
$peers[] = $peer;
}
}
return $peers;
}
}
return false;
}
}

32
curl/torproject.php Normal file
View File

@ -0,0 +1,32 @@
<?php
class CurlTorProject extends Curl {
public function getExitNodes() {
$this->prepare('torbulkexitlist', 'GET');
if ($response = $this->execute(false)) {
$list = explode("\n", $response);
if (count($list)) {
$sanitized = [];
foreach ($list as $ip) {
$ip = $this->sanitize($ip);
if ($ip) {
$sanitized[] = $ip;
}
}
return $sanitized;
}
}
return false;
}
}

176
model/ip.php Normal file
View File

@ -0,0 +1,176 @@
<?php
class ModelIp extends Model {
public function exists($address) {
try {
$query = $this->db->prepare('SELECT `ipId` FROM `ip` WHERE `address` = ? LIMIT 1');
$query->execute([$address]);
return $query->rowCount() ? $query->fetch()['ipId'] : false;
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getIps() {
try {
$query = $this->db->prepare('SELECT `ipId`, `address` FROM `ip`');
$query->execute();
return $query->rowCount() ? $query->fetchAll() : [];
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getIsOnlineIps() {
try {
$query = $this->db->prepare('SELECT `address` FROM `ip` WHERE `isOnline` = "1"');
$query->execute();
return $query->rowCount() ? $query->fetchAll() : [];
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function getIsOfflineIps() {
try {
$query = $this->db->prepare('SELECT `address` FROM `ip` WHERE `isOnline` = "0"');
$query->execute();
return $query->rowCount() ? $query->fetchAll() : [];
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function add($address, $port) {
try {
$query = $this->db->prepare('INSERT INTO `ip` SET `address` = ?, `port` = ?, `isOnline` = "0", `isTOR` = "0"');
$query->execute([$address, $port]);
return $this->db->lastInsertId();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function addOnline($ipId, $startingHeight, $timeConnection, $timeLastSend, $timeLastReceive) {
try {
$query = $this->db->prepare('INSERT INTO `ipOnline` SET `ipId` = ?,
`startingHeight` = ?,
`timeConnection` = ?,
`timeLastSend` = ?,
`timeLastReceive` = ?,
`timeAdded` = UNIX_TIMESTAMP()');
$query->execute([$ipId, $startingHeight, $timeConnection, $timeLastSend, $timeLastReceive]);
return $this->db->lastInsertId();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function updateGeoData($ipId, $countryCode, $city, $latitude, $longitude) {
try {
$query = $this->db->prepare('UPDATE `ip` SET `countryCode` = ?,
`city` = ?,
`latitude` = ?,
`longitude` = ?
WHERE `ipId` = ?
LIMIT 1');
$query->execute([$countryCode, $city, $latitude, $longitude, $ipId]);
return $query->rowCount();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function resetIsOnline() {
try {
$query = $this->db->prepare('UPDATE `ip` SET `isOnline` = "0"');
$query->execute();
return $query->rowCount();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function setIsOnline($ipId) {
try {
$query = $this->db->prepare('UPDATE `ip` SET `isOnline` = "1" WHERE `ipId` = ? LIMIT 1');
$query->execute([$ipId]);
return $query->rowCount();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
public function setIsTOR($ipId) {
try {
$query = $this->db->prepare('UPDATE `ip` SET `isTOR` = "1" WHERE `ipId` = ? LIMIT 1');
$query->execute([$ipId]);
return $query->rowCount();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
}

20
model/log.php Normal file
View File

@ -0,0 +1,20 @@
<?php
class ModelLog extends Model {
public function add($message) {
try {
$query = $this->db->prepare('INSERT INTO `log` SET `message` = ?, `timeAdded` = UNIX_TIMESTAMP()');
$query->execute([$message]);
return $this->db->lastInsertId();
} catch (PDOException $e) {
trigger_error($e->getMessage());
return false;
}
}
}

16
model/model.php Normal file
View File

@ -0,0 +1,16 @@
<?php
class Model {
protected $db;
public function __construct($database, $hostname, $port, $user, $password) {
try {
$this->db = new PDO('mysql:dbname=' . $database . ';host=' . $hostname . ';port=' . $port . ';charset=utf8', $user, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_ASSOC);
} catch(PDOException $e) {
trigger_error($e->getMessage());
}
}
}

119
peer.php Normal file
View File

@ -0,0 +1,119 @@
<?php
require(__DIR__ . '/config.php');
require(__DIR__ . '/curl/curl.php');
require(__DIR__ . '/curl/peer.php');
require(__DIR__ . '/curl/geoplugin.php');
require(__DIR__ . '/model/model.php');
require(__DIR__ . '/model/ip.php');
require(__DIR__ . '/model/log.php');
$curlPeer = new CurlPeer(TWISTER_PROTOCOL, TWISTER_HOST, TWISTER_PORT, TWISTER_USERNAME, TWISTER_PASSWORD);
$curlGeoPlugin = new CurlGeoPlugin(GEOPLUGIN_PROTOCOL, GEOPLUGIN_HOST, GEOPLUGIN_PORT);
$modelIp = new ModelIp(DB_DATABASE, DB_HOSTNAME, DB_PORT, DB_USERNAME, DB_PASSWORD);
$modelLog = new ModelLog(DB_DATABASE, DB_HOSTNAME, DB_PORT, DB_USERNAME, DB_PASSWORD);
$isOnlinePeers = [];
$isOfflinePeers = [];
$toOnlinePeers = [];
$toOfflinePeers = [];
$newPeers = [];
$onlinePeers = [];
// Get online peers list
foreach ($modelIp->getIsOnlineIps() as $isOnlinePeer) {
$isOnlinePeers[] = $isOnlinePeer['address'];
}
// Get offline peers list
foreach ($modelIp->getIsOfflineIps() as $isOfflinePeer) {
$isOfflinePeers[] = $isOfflinePeer['address'];
}
// Reset peers online
$modelIp->resetIsOnline();
// Get current peers
if ($peers = $curlPeer->getAll()) {
foreach ($peers as $peer) {
if (isset($peer['addr'])) {
// Parse response
if (false !== preg_match('/(.*):(\d+)$/', $peer['addr'], $matches)) {
if (isset($matches[1]) && isset($matches[2])) {
// IP exist
if (!$ipId = $modelIp->exists($matches[1])) {
// Save IP
$ipId = $modelIp->add($matches[1], $matches[2]);
// Get geo info
if ($location = $curlGeoPlugin->getLocation($matches[1])) {
$modelIp->updateGeoData($ipId,
$location['geoplugin_countryCode'],
$location['geoplugin_city'],
$location['geoplugin_latitude'],
$location['geoplugin_longitude']);
} else {
$modelLog->add(_('Could not receive geolocation details'));
}
$newPeers[] = $matches[1];
}
// Peer switching online
if (in_array($matches[1], $isOfflinePeers)) {
$toOnlinePeers[] = $matches[1];
}
// Add online peers to registry
$onlinePeers[] = $matches[1];
// Set peer as online
$modelIp->setIsOnline($ipId);
// Update online time
$modelIp->addOnline($ipId,
$peer['startingheight'],
$peer['conntime'],
$peer['lastsend'],
$peer['lastrecv']);
} else {
$modelLog->add(_('Could not extract peer address or port'));
}
} else {
$modelLog->add(_('Could not parse peer address'));
}
} else {
$modelLog->add(_('Could not parse RPC response'));
}
}
} else {
$modelLog->add(_('Could not connect to twister peer'));
}
// Alert if peer(s) going to offline
$toOfflinePeers = array_diff($isOnlinePeers, $onlinePeers);
if (EMAIL_OFFLINE_PEERS && $toOfflinePeers) {
mail(EMAIL_OFFLINE_PEERS, sprintf(_('Peer(s) switched to offline')), implode("\r\n", $toOfflinePeers));
}
// Alert if peer(s) going to online
if (EMAIL_ONLINE_PEERS && $toOnlinePeers) {
mail(EMAIL_ONLINE_PEERS, sprintf(_('Peer(s) switched to online')), implode("\r\n", $toOnlinePeers));
}
// Alert if new peer(s) available
if (EMAIL_NEW_PEERS && $newPeers) {
mail(EMAIL_NEW_PEERS, sprintf(_('New peer(s) added')), implode("\r\n", $newPeers));
}

37
tor.php Normal file
View File

@ -0,0 +1,37 @@
<?php
require(__DIR__ . '/config.php');
require(__DIR__ . '/curl/curl.php');
require(__DIR__ . '/curl/torproject.php');
require(__DIR__ . '/model/model.php');
require(__DIR__ . '/model/ip.php');
require(__DIR__ . '/model/log.php');
$curlTorProject = new CurlTorProject(TORPROJECT_PROTOCOL, TORPROJECT_HOST, TORPROJECT_PORT);
$modelIp = new ModelIp(DB_DATABASE, DB_HOSTNAME, DB_PORT, DB_USERNAME, DB_PASSWORD);
$modelLog = new ModelLog(DB_DATABASE, DB_HOSTNAME, DB_PORT, DB_USERNAME, DB_PASSWORD);
$exitNodes = [];
// Get TOR registry
if ($torProjectExitNodes = $curlTorProject->getExitNodes()) {
foreach ($torProjectExitNodes as $exitNode) {
$exitNodes[] = $exitNode;
}
// Get IPs
foreach ($modelIp->getIps() as $ip) {
if (in_array($ip['address'], $exitNodes)) {
$modelIp->setIsTOR($ip['ipId']);
}
}
} else {
$modelLog->add(_('Could not parse TorProject response'));
}