diff --git a/.gitignore b/.gitignore new file mode 100644 index 0000000..800c6ab --- /dev/null +++ b/.gitignore @@ -0,0 +1,3 @@ +/venv +results.json +yggcrawl.log diff --git a/api/crawler.py b/api/crawler.py index 7c46f93..6b646ba 100644 --- a/api/crawler.py +++ b/api/crawler.py @@ -13,12 +13,14 @@ import traceback from threading import Lock, Thread from queue import Queue +##### + # Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil useAdminSock = True yggAdminTCP = ('localhost', 9001) yggAdminSock = ('/var/run/yggdrasil.sock') -DB_PASSWORD = "password" +DB_PASS = "password" DB_USER = "yggindex" DB_NAME = "yggindex" DB_HOST = "localhost" @@ -27,6 +29,8 @@ DB_HOST = "localhost" saveDefaultNodeInfo = False removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system'] +##### + class Worker(Thread): def __init__(self, tasks): Thread.__init__(self) @@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords): nodejson = json.dumps(nodeinfo[ipv6]) nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else "" - dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD) + dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS) cur = dbconn.cursor() timestamp = str(int(time.time())) diff --git a/api/importer.py b/api/importer.py new file mode 100644 index 0000000..c3eafc7 --- /dev/null +++ b/api/importer.py @@ -0,0 +1,69 @@ +#!/usr/bin/env python + +import psycopg2, json, traceback + +##### + +# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil +DB_PASS = "password" +DB_USER = "yggindex" +DB_NAME = "yggindex" +DB_HOST = "localhost" + +## Save in database node info fields like buildname, buildarch, etc. (True/False)? +saveDefaultNodeInfo = False +removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system'] + +##### + +with open('api/results.json', 'r') as f: + data = json.load(f) + +timestamp = data['meta']['generated_at_utc'] + +# connect to database +dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS) +cur = dbconn.cursor() + +# start importing +for node in data['topology']: + nodename = "" + nodeinfo = {} + ipv6 = data['topology'][node]['ipv6_addr'] + coords = '[%s]' % (' '.join(str(e) for e in data['topology'][node]['coords'])) + + if node in data['nodeinfo']: + nodeinfo = data['nodeinfo'][node] + + if not saveDefaultNodeInfo: + # remove default Node info fields + for field in removableFileds: + tmprm = nodeinfo.pop(field, None) + + if "name" in nodeinfo: + nodename = nodeinfo['name'] + elif data['topology'][node]['found'] == False: + nodename = '? %s' % coords + else: + nodename = ipv6 + + nodeinfo = json.dumps(nodeinfo) + + try: + cur.execute( + "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;", + (ipv6, coords, timestamp, nodename, timestamp, coords, nodename) + ) + cur.execute( + "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;", + (ipv6, nodeinfo, timestamp, nodeinfo, timestamp) + ) + + except Exception as e: + print("database error inserting") + traceback.print_exc() + +dbconn.commit() +cur.close() +dbconn.close() + diff --git a/api/max-min.py b/api/max-min.py index 5e81515..d14e8bd 100644 --- a/api/max-min.py +++ b/api/max-min.py @@ -7,7 +7,7 @@ import time #run every hour -DB_PASSWORD = "password" +DB_PASS = "password" DB_USER = "yggindex" DB_NAME = "yggindex" DB_HOST = "localhost" @@ -24,7 +24,7 @@ def age_calc(ustamp): return False def get_nodes_for_count(): - dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD) + dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS) cur = dbconn.cursor() nodes = {} cur.execute("select * from yggindex") @@ -39,7 +39,7 @@ def get_nodes_for_count(): return str(len(nodes)) def add_to_db(): - dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD) + dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS) cur = dbconn.cursor() cur.execute('''INSERT INTO timeseries(max, unixtstamp) VALUES(''' + "'" + get_nodes_for_count() + "'," + str(int(time.time())) + ''')''') diff --git a/api/niflheim-api.py b/api/niflheim-api.py index bb6a3aa..6e4aa27 100644 --- a/api/niflheim-api.py +++ b/api/niflheim-api.py @@ -1,16 +1,15 @@ #!/usr/bin/env python -import time +import signal, sys, time from flask import Flask, render_template from flask_restful import Resource, Api import requests import psycopg2 import json -app = Flask(__name__) -api = Api(app) +###### -DB_PASSWORD = "password" +DB_PASS = "password" DB_USER = "yggindex" DB_NAME = "yggindex" DB_HOST = "localhost" @@ -19,6 +18,19 @@ DB_HOST = "localhost" # I'm using 1 hour beause of running crawler every 15 minutes ALIVE_SECONDS = 3600 # 1 hour +###### + +app = Flask(__name__) +api = Api(app) + +dbconn = psycopg2.connect(host=DB_HOST,\ + database=DB_NAME,\ + user=DB_USER,\ + password=DB_PASS) + +def signal_handler(sig, frame): + dbconn.close() + sys.exit(0) def age_calc(ustamp): if (time.time() - ustamp) <= ALIVE_SECONDS: @@ -29,10 +41,6 @@ def age_calc(ustamp): # active nodes class nodesCurrent(Resource): def get(self): - dbconn = psycopg2.connect(host=DB_HOST,\ - database=DB_NAME,\ - user=DB_USER,\ - password=DB_PASSWORD) cur = dbconn.cursor() nodes = {} cur.execute("select * from yggindex") @@ -42,7 +50,6 @@ class nodesCurrent(Resource): dbconn.commit() cur.close() - dbconn.close() nodelist = {} nodelist['yggnodes'] = nodes @@ -53,10 +60,6 @@ class nodesCurrent(Resource): # nodes info class nodesInfo(Resource): def get(self): - dbconn = psycopg2.connect(host=DB_HOST,\ - database=DB_NAME,\ - user=DB_USER,\ - password=DB_PASSWORD) cur = dbconn.cursor() nodes = {} cur.execute("select * from yggnodeinfo") @@ -66,7 +69,6 @@ class nodesInfo(Resource): dbconn.commit() cur.close() - dbconn.close() nodeinfo = {} nodeinfo['yggnodeinfo'] = nodes @@ -77,10 +79,6 @@ class nodesInfo(Resource): # alive nodes count for latest 24 hours class nodes24h(Resource): def get(self): - dbconn = psycopg2.connect(host=DB_HOST,\ - database=DB_NAME,\ - user=DB_USER,\ - password=DB_PASSWORD) cur = dbconn.cursor() nodes = {} cur.execute("SELECT * FROM timeseries ORDER BY unixtstamp DESC LIMIT 24") @@ -89,21 +87,23 @@ class nodes24h(Resource): dbconn.commit() cur.close() - dbconn.close() nodeinfo = {} nodeinfo['nodes24h'] = nodes return nodeinfo +# alive nodes count for latest 24 hours +class crawlResult(Resource): + def get(self): + with open('api/results.json', 'r') as f: + data = json.load(f) + + return data @app.route("/") def fpage(): - dbconn = psycopg2.connect(host=DB_HOST,\ - database=DB_NAME,\ - user=DB_USER,\ - password=DB_PASSWORD) cur = dbconn.cursor() nodes = 0 cur.execute("select * from yggindex") @@ -114,15 +114,19 @@ def fpage(): dbconn.commit() cur.close() - dbconn.close() return render_template('index.html', nodes=nodes) -#sort out the api request here for the url +# sort out the api request here for the url api.add_resource(nodesCurrent, '/current') api.add_resource(nodesInfo, '/nodeinfo') api.add_resource(nodes24h, '/nodes24h') +api.add_resource(crawlResult, '/result.json') + +# regirster signal handler +signal.signal(signal.SIGINT, signal_handler) +signal.signal(signal.SIGTERM, signal_handler) if __name__ == '__main__': app.run(host='::', port=3000) diff --git a/api/templates/index.html b/api/templates/index.html index cebb8dc..56c96c7 100644 --- a/api/templates/index.html +++ b/api/templates/index.html @@ -13,9 +13,10 @@
Current Nodes online
{{ nodes }} +

+ You can see them in Yggdrasil Interactive World map [⇗] [DNS]


- Yggdrasil Interactive World map

Make an API request
@@ -23,17 +24,22 @@ Get a current list of active and online nodes:
- http://[31a:fb8a:c43e:ca59::2]/current ⇗ + http://[31a:fb8a:c43e:ca59::2]/current [⇗] [DNS]
Nodeinfo from all current active nodes:
- http://[31a:fb8a:c43e:ca59::2]/nodeinfo ⇗ + http://[31a:fb8a:c43e:ca59::2]/nodeinfo [⇗] [DNS]
Active nodes count for last 24 hours:
- http://[31a:fb8a:c43e:ca59::2]/nodes24h ⇗ + http://[31a:fb8a:c43e:ca59::2]/nodes24h [⇗] [DNS]
+
+ Get latest crawler data
+ You can download data in raw json provided by yggcrawl: result.json [⇗] [DNS] +

+
Made with fork of Niflheim-API by yakamok diff --git a/ygg-crawl.sh b/ygg-crawl.sh new file mode 100755 index 0000000..17a8ae8 --- /dev/null +++ b/ygg-crawl.sh @@ -0,0 +1,12 @@ +#!/bin/sh + +YGGCRAWL="/opt/yggcrawl/yggcrawl" # path to yggcrawl binary +YGGAPIPATH="/opt/yggdrasil-api" # path to Niflheim-API directory + +CRAWLPEER="tcp://127.0.0.1:12345" # Yggdrasil peer address +CRAWLFILE="api/results.json" +CRAWLRETR=3 + +cd $YGGAPIPATH +$YGGCRAWL -peer $CRAWLPEER -retry $CRAWLRETR -file $CRAWLFILE > api/yggcrawl.log 2>&1 +venv/bin/python api/importer.py >> api/yggcrawl.log 2>&1