Browse Source

add importer from yggcrawl, update template

master
R4SAS 4 years ago
parent
commit
a37f5b0ec4
  1. 3
      .gitignore
  2. 8
      api/crawler.py
  3. 69
      api/importer.py
  4. 6
      api/max-min.py
  5. 54
      api/niflheim-api.py
  6. 14
      api/templates/index.html
  7. 12
      ygg-crawl.sh

3
.gitignore vendored

@ -0,0 +1,3 @@ @@ -0,0 +1,3 @@
/venv
results.json
yggcrawl.log

8
api/crawler.py

@ -13,12 +13,14 @@ import traceback @@ -13,12 +13,14 @@ import traceback
from threading import Lock, Thread
from queue import Queue
#####
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
useAdminSock = True
yggAdminTCP = ('localhost', 9001)
yggAdminSock = ('/var/run/yggdrasil.sock')
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -27,6 +29,8 @@ DB_HOST = "localhost" @@ -27,6 +29,8 @@ DB_HOST = "localhost"
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
#####
class Worker(Thread):
def __init__(self, tasks):
Thread.__init__(self)
@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords): @@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords):
nodejson = json.dumps(nodeinfo[ipv6])
nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
timestamp = str(int(time.time()))

69
api/importer.py

@ -0,0 +1,69 @@ @@ -0,0 +1,69 @@
#!/usr/bin/env python
import psycopg2, json, traceback
#####
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
## Save in database node info fields like buildname, buildarch, etc. (True/False)?
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
#####
with open('api/results.json', 'r') as f:
data = json.load(f)
timestamp = data['meta']['generated_at_utc']
# connect to database
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
# start importing
for node in data['topology']:
nodename = ""
nodeinfo = {}
ipv6 = data['topology'][node]['ipv6_addr']
coords = '[%s]' % (' '.join(str(e) for e in data['topology'][node]['coords']))
if node in data['nodeinfo']:
nodeinfo = data['nodeinfo'][node]
if not saveDefaultNodeInfo:
# remove default Node info fields
for field in removableFileds:
tmprm = nodeinfo.pop(field, None)
if "name" in nodeinfo:
nodename = nodeinfo['name']
elif data['topology'][node]['found'] == False:
nodename = '? %s' % coords
else:
nodename = ipv6
nodeinfo = json.dumps(nodeinfo)
try:
cur.execute(
"INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
(ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
)
cur.execute(
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
(ipv6, nodeinfo, timestamp, nodeinfo, timestamp)
)
except Exception as e:
print("database error inserting")
traceback.print_exc()
dbconn.commit()
cur.close()
dbconn.close()

6
api/max-min.py

@ -7,7 +7,7 @@ import time @@ -7,7 +7,7 @@ import time
#run every hour
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -24,7 +24,7 @@ def age_calc(ustamp): @@ -24,7 +24,7 @@ def age_calc(ustamp):
return False
def get_nodes_for_count():
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggindex")
@ -39,7 +39,7 @@ def get_nodes_for_count(): @@ -39,7 +39,7 @@ def get_nodes_for_count():
return str(len(nodes))
def add_to_db():
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
cur = dbconn.cursor()
cur.execute('''INSERT INTO timeseries(max, unixtstamp) VALUES(''' + "'" + get_nodes_for_count() + "'," + str(int(time.time())) + ''')''')

54
api/niflheim-api.py

@ -1,16 +1,15 @@ @@ -1,16 +1,15 @@
#!/usr/bin/env python
import time
import signal, sys, time
from flask import Flask, render_template
from flask_restful import Resource, Api
import requests
import psycopg2
import json
app = Flask(__name__)
api = Api(app)
######
DB_PASSWORD = "password"
DB_PASS = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
@ -19,6 +18,19 @@ DB_HOST = "localhost" @@ -19,6 +18,19 @@ DB_HOST = "localhost"
# I'm using 1 hour beause of running crawler every 15 minutes
ALIVE_SECONDS = 3600 # 1 hour
######
app = Flask(__name__)
api = Api(app)
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASS)
def signal_handler(sig, frame):
dbconn.close()
sys.exit(0)
def age_calc(ustamp):
if (time.time() - ustamp) <= ALIVE_SECONDS:
@ -29,10 +41,6 @@ def age_calc(ustamp): @@ -29,10 +41,6 @@ def age_calc(ustamp):
# active nodes
class nodesCurrent(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggindex")
@ -42,7 +50,6 @@ class nodesCurrent(Resource): @@ -42,7 +50,6 @@ class nodesCurrent(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodelist = {}
nodelist['yggnodes'] = nodes
@ -53,10 +60,6 @@ class nodesCurrent(Resource): @@ -53,10 +60,6 @@ class nodesCurrent(Resource):
# nodes info
class nodesInfo(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggnodeinfo")
@ -66,7 +69,6 @@ class nodesInfo(Resource): @@ -66,7 +69,6 @@ class nodesInfo(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodeinfo = {}
nodeinfo['yggnodeinfo'] = nodes
@ -77,10 +79,6 @@ class nodesInfo(Resource): @@ -77,10 +79,6 @@ class nodesInfo(Resource):
# alive nodes count for latest 24 hours
class nodes24h(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("SELECT * FROM timeseries ORDER BY unixtstamp DESC LIMIT 24")
@ -89,21 +87,23 @@ class nodes24h(Resource): @@ -89,21 +87,23 @@ class nodes24h(Resource):
dbconn.commit()
cur.close()
dbconn.close()
nodeinfo = {}
nodeinfo['nodes24h'] = nodes
return nodeinfo
# alive nodes count for latest 24 hours
class crawlResult(Resource):
def get(self):
with open('api/results.json', 'r') as f:
data = json.load(f)
return data
@app.route("/")
def fpage():
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = 0
cur.execute("select * from yggindex")
@ -114,15 +114,19 @@ def fpage(): @@ -114,15 +114,19 @@ def fpage():
dbconn.commit()
cur.close()
dbconn.close()
return render_template('index.html', nodes=nodes)
#sort out the api request here for the url
# sort out the api request here for the url
api.add_resource(nodesCurrent, '/current')
api.add_resource(nodesInfo, '/nodeinfo')
api.add_resource(nodes24h, '/nodes24h')
api.add_resource(crawlResult, '/result.json')
# regirster signal handler
signal.signal(signal.SIGINT, signal_handler)
signal.signal(signal.SIGTERM, signal_handler)
if __name__ == '__main__':
app.run(host='::', port=3000)

14
api/templates/index.html

@ -13,9 +13,10 @@ @@ -13,9 +13,10 @@
<center>
<strong>Current Nodes online<br />
<font size="18">{{ nodes }}</font></strong>
<br /><br />
You can see them in <i>Yggdrasil Interactive World map</i> <sup><a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">[&#8663;]</a> <a href="http://map.ygg/" target="_blank">[DNS]</a></sup>
</center>
<br /><br />
<a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
<div class="wide"></div>
<strong>Make an API request</strong><br />
@ -23,17 +24,22 @@ @@ -23,17 +24,22 @@
Get a current list of active and online nodes:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="/current" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="http://[31a:fb8a:c43e:ca59::2]/current" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/current" target="_blank">[DNS]</a></sup>
</div>
Nodeinfo from all current active nodes:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="/nodeinfo" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodeinfo" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodeinfo" target="_blank">[DNS]</a></sup>
</div>
Active nodes count for last 24 hours:<br />
<div class="apireq">
http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="/nodes24h" target="_blank">&#8663;</a></sup>
http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodes24h" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodes24h" target="_blank">[DNS]</a></sup>
</div>
<div class="wide"></div>
<strong>Get latest crawler data</strong><br />
You can download data in raw json provided by <a href="https://github.com/neilalexander/yggcrawl" target="_blank">yggcrawl</a>: result.json <sup><a href="http://[31a:fb8a:c43e:ca59::2]/result.json" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/result.json" target="_blank">[DNS]</a></sup>
<br /><br />
<div class="wide"></div>
<small>Made with <a href="https://github.com/r4sas/Niflheim-api" target="_blank">fork</a> of <a href="https://github.com/yakamok/Niflheim-api" target="_blank">Niflheim-API</a> by yakamok</small>
</div>

12
ygg-crawl.sh

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
#!/bin/sh
YGGCRAWL="/opt/yggcrawl/yggcrawl" # path to yggcrawl binary
YGGAPIPATH="/opt/yggdrasil-api" # path to Niflheim-API directory
CRAWLPEER="tcp://127.0.0.1:12345" # Yggdrasil peer address
CRAWLFILE="api/results.json"
CRAWLRETR=3
cd $YGGAPIPATH
$YGGCRAWL -peer $CRAWLPEER -retry $CRAWLRETR -file $CRAWLFILE > api/yggcrawl.log 2>&1
venv/bin/python api/importer.py >> api/yggcrawl.log 2>&1
Loading…
Cancel
Save