r4sas
/
yggdrasil-api
mirror of https://github.com/r4sas/Niflheim-api
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

add importer from yggcrawl, update template

master
R4SAS 5 years ago
parent
e68961f131
commit
a37f5b0ec4
7 changed files with 132 additions and 34 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 3
      .gitignore
  2. 8
      api/crawler.py
  3. 69
      api/importer.py
  4. 6
      api/max-min.py
  5. 52
      api/niflheim-api.py
  6. 14
      api/templates/index.html
  7. 12
      ygg-crawl.sh

3
.gitignore vendored
Unescape View File

@ -0,0 +1,3 @@ @@ -0,0 +1,3 @@
/venv
results.json
yggcrawl.log

8
api/crawler.py
Unescape View File

@ -13,12 +13,14 @@ import traceback @@ -13,12 +13,14 @@ import traceback
from threading import Lock, Thread
from queue import Queue
#####
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
useAdminSock = True
yggAdminTCP = ('localhost', 9001)
yggAdminSock = ('/var/run/yggdrasil.sock')
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -27,6 +29,8 @@ DB_HOST = "localhost" @@ -27,6 +29,8 @@ DB_HOST = "localhost"
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
#####
class Worker(Thread):
def __init__(self, tasks):
Thread.__init__(self)
@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords): @@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords):
nodejson = json.dumps(nodeinfo[ipv6])
nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
timestamp = str(int(time.time()))

69
api/importer.py
Unescape View File

@ -0,0 +1,69 @@ @@ -0,0 +1,69 @@
#!/usr/bin/env python
import psycopg2, json, traceback
#####
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
## Save in database node info fields like buildname, buildarch, etc. (True/False)?
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
#####
with open('api/results.json', 'r') as f:
data = json.load(f)
timestamp = data['meta']['generated_at_utc']
# connect to database
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
# start importing
for node in data['topology']:
nodename = ""
nodeinfo = {}
ipv6 = data['topology'][node]['ipv6_addr']
coords = '[%s]' % (' '.join(str(e) for e in data['topology'][node]['coords']))
if node in data['nodeinfo']:
nodeinfo = data['nodeinfo'][node]
if not saveDefaultNodeInfo:
# remove default Node info fields
for field in removableFileds:
tmprm = nodeinfo.pop(field, None)
if "name" in nodeinfo:
nodename = nodeinfo['name']
elif data['topology'][node]['found'] == False:
nodename = '? %s' % coords
else:
nodename = ipv6
nodeinfo = json.dumps(nodeinfo)
try:
cur.execute(
"INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
(ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
)
cur.execute(
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
(ipv6, nodeinfo, timestamp, nodeinfo, timestamp)
)
except Exception as e:
print("database error inserting")
traceback.print_exc()
dbconn.commit()
cur.close()
dbconn.close()

6
api/max-min.py
Unescape View File

@ -7,7 +7,7 @@ import time @@ -7,7 +7,7 @@ import time
#run every hour
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -24,7 +24,7 @@ def age_calc(ustamp): @@ -24,7 +24,7 @@ def age_calc(ustamp):
return False
def get_nodes_for_count():
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggindex")
@ -39,7 +39,7 @@ def get_nodes_for_count(): @@ -39,7 +39,7 @@ def get_nodes_for_count():
return str(len(nodes))
def add_to_db():
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
cur.execute('''INSERT INTO timeseries(max, unixtstamp) VALUES(''' + "'" + get_nodes_for_count() + "'," + str(int(time.time())) + ''')''')

52
api/niflheim-api.py
Unescape View File

@ -1,16 +1,15 @@ @@ -1,16 +1,15 @@
#!/usr/bin/env python
import time
import signal, sys, time
from flask import Flask, render_template
from flask_restful import Resource, Api
import requests
import psycopg2
import json
app = Flask(__name__)
api = Api(app)
######
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -19,6 +18,19 @@ DB_HOST = "localhost" @@ -19,6 +18,19 @@ DB_HOST = "localhost"
# I'm using 1 hour beause of running crawler every 15 minutes
ALIVE_SECONDS = 3600 # 1 hour
######
app = Flask(__name__)
api = Api(app)
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASS)
def signal_handler(sig, frame):
dbconn.close()
sys.exit(0)
def age_calc(ustamp):
if (time.time() - ustamp) <= ALIVE_SECONDS:
@ -29,10 +41,6 @@ def age_calc(ustamp): @@ -29,10 +41,6 @@ def age_calc(ustamp):
# active nodes
class nodesCurrent(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggindex")
@ -42,7 +50,6 @@ class nodesCurrent(Resource): @@ -42,7 +50,6 @@ class nodesCurrent(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodelist = {}
nodelist['yggnodes'] = nodes
@ -53,10 +60,6 @@ class nodesCurrent(Resource): @@ -53,10 +60,6 @@ class nodesCurrent(Resource):
# nodes info
class nodesInfo(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggnodeinfo")
@ -66,7 +69,6 @@ class nodesInfo(Resource): @@ -66,7 +69,6 @@ class nodesInfo(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodeinfo = {}
nodeinfo['yggnodeinfo'] = nodes
@ -77,10 +79,6 @@ class nodesInfo(Resource): @@ -77,10 +79,6 @@ class nodesInfo(Resource):
# alive nodes count for latest 24 hours
class nodes24h(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("SELECT * FROM timeseries ORDER BY unixtstamp DESC LIMIT 24")
@ -89,21 +87,23 @@ class nodes24h(Resource): @@ -89,21 +87,23 @@ class nodes24h(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodeinfo = {}
nodeinfo['nodes24h'] = nodes
return nodeinfo
# alive nodes count for latest 24 hours
class crawlResult(Resource):
def get(self):
with open('api/results.json', 'r') as f:
data = json.load(f)
return data
@app.route("/")
def fpage():
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = 0
cur.execute("select * from yggindex")
@ -114,7 +114,6 @@ def fpage(): @@ -114,7 +114,6 @@ def fpage():
dbconn.commit()
cur.close()
dbconn.close()
return render_template('index.html', nodes=nodes)
@ -123,6 +122,11 @@ def fpage(): @@ -123,6 +122,11 @@ def fpage():
api.add_resource(nodesCurrent, '/current')
api.add_resource(nodesInfo, '/nodeinfo')
api.add_resource(nodes24h, '/nodes24h')
api.add_resource(crawlResult, '/result.json')
# regirster signal handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
if __name__ == '__main__':
app.run(host='::', port=3000)

14
api/templates/index.html
Unescape View File

@ -13,9 +13,10 @@ @@ -13,9 +13,10 @@
<center>
<strong>Current Nodes online<br />
<font size="18">{{ nodes }}</font></strong>
<br /><br />
You can see them in <i>Yggdrasil Interactive World map</i> <sup><a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">[&#8663;]</a> <a href="http://map.ygg/" target="_blank">[DNS]</a></sup>
</center>
<br /><br />
<a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
<div class="wide"></div>
<strong>Make an API request</strong><br />
@ -23,17 +24,22 @@ @@ -23,17 +24,22 @@
Get a current list of active and online nodes:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="/current" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="http://[31a:fb8a:c43e:ca59::2]/current" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/current" target="_blank">[DNS]</a></sup>
</div>
Nodeinfo from all current active nodes:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="/nodeinfo" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodeinfo" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodeinfo" target="_blank">[DNS]</a></sup>
</div>
Active nodes count for last 24 hours:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="/nodes24h" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodes24h" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodes24h" target="_blank">[DNS]</a></sup>
</div>
<div class="wide"></div>
<strong>Get latest crawler data</strong><br />
You can download data in raw json provided by <a href="https://github.com/neilalexander/yggcrawl" target="_blank">yggcrawl</a>: result.json <sup><a href="http://[31a:fb8a:c43e:ca59::2]/result.json" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/result.json" target="_blank">[DNS]</a></sup>
<br /><br />
<div class="wide"></div>
<small>Made with <a href="https://github.com/r4sas/Niflheim-api" target="_blank">fork</a> of <a href="https://github.com/yakamok/Niflheim-api" target="_blank">Niflheim-API</a> by yakamok</small>
</div>

12
ygg-crawl.sh
Unescape View File

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
#!/bin/sh
YGGCRAWL="/opt/yggcrawl/yggcrawl" # path to yggcrawl binary
YGGAPIPATH="/opt/yggdrasil-api" # path to Niflheim-API directory
CRAWLPEER="tcp://127.0.0.1:12345" # Yggdrasil peer address
CRAWLFILE="api/results.json"
CRAWLRETR=3
cd $YGGAPIPATH
$YGGCRAWL -peer $CRAWLPEER -retry $CRAWLRETR -file $CRAWLFILE > api/yggcrawl.log 2>&1
venv/bin/python api/importer.py >> api/yggcrawl.log 2>&1
Write Preview
Loading…
Cancel
Save