diff --git a/api/crawler.py b/api/crawler.py index 3bf00f1..d3c7f37 100644 --- a/api/crawler.py +++ b/api/crawler.py @@ -1,6 +1,8 @@ -#some of this code was contributed by Arcelier -#original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py -#multithreaded by neilalexander +#!/usr/bin/env python + +# some of this code was contributed by Arcelier +# original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py +# multithreaded by neilalexander import psycopg2 import json @@ -12,6 +14,20 @@ import traceback from threading import Lock, Thread from Queue import Queue +# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil +useAdminSock = True +yggAdminTCP = ('localhost', 9001) +yggAdminSock = ('/var/run/yggdrasil.sock') + +DB_PASSWORD = "password" +DB_USER = "yggindex" +DB_NAME = "yggindex" +DB_HOST = "localhost" + +## Save in database node info fields like buildname, buildarch, buildplatform, buildversion (True/False)? +saveDefaultNodeInfo = False +removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion'] + class Worker(Thread): def __init__(self, tasks): Thread.__init__(self) @@ -48,13 +64,6 @@ nodeinfo = dict() nodeinfomutex = Lock() nodeinfopool = ThreadPool(30) -host_port = ('localhost', 9001) - -DB_PASSWORD = "akemi2501" -DB_USER = "yggindex" -DB_NAME = "yggindex" -DB_HOST = "localhost" - def recv_until_done(soc): all_data = [] while True: @@ -83,8 +92,13 @@ def getNodeInfoTask(address, info): def doRequest(req): try: - ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) - ygg.connect(host_port) + if useAdminSock: + ygg = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM) + ygg.connect(yggAdminSock) + else: + ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) + ygg.connect(yggAdminTCP) + ygg.send(req) data = json.loads(recv_until_done(ygg)) return data @@ -118,24 +132,34 @@ def insert_new_entry(ipv6, coords): nodejson = "{}" if ipv6 in nodeinfo: with nodeinfomutex: + + if not saveDefaultNodeInfo: + # remove default Node info fields + for field in removableFileds: + tmprm = nodeinfo[ipv6].pop(field, None) + nodejson = json.dumps(nodeinfo[ipv6]) nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else "" + dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD) cur = dbconn.cursor() + timestamp = str(int(time.time())) + cur.execute( "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;", - (ipv6, coords, str(int(time.time())), nodename, str(int(time.time())), coords, nodename) + (ipv6, coords, timestamp, nodename, timestamp, coords, nodename) ) cur.execute( - "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, NOW()) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=NOW();", - (ipv6, nodejson, nodejson) + "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;", + (ipv6, nodejson, timestamp, nodejson, timestamp) ) + dbconn.commit() cur.close() dbconn.close() except Exception as e: - print("database error inserting") - traceback.print_exc() + print "database error inserting" + traceback.print_exc() def handleNodeInfo(address, data): global nodeinfo @@ -183,6 +207,7 @@ for k,v in selfInfo['response']['self'].iteritems(): # Loop over rumored nodes and ping them, adding to visited if they respond while len(rumored) > 0: for k,v in rumored.iteritems(): + #print "Processing", v['coords'] handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords']))) break del rumored[k] diff --git a/api/max-min.py b/api/max-min.py index 578912b..bc86ccf 100644 --- a/api/max-min.py +++ b/api/max-min.py @@ -1,4 +1,7 @@ +#!/usr/bin/env python + #max/min for the day with nodes + import psycopg2 import time diff --git a/api/niflheim-api.py b/api/niflheim-api.py index 8fea46a..396c867 100644 --- a/api/niflheim-api.py +++ b/api/niflheim-api.py @@ -1,8 +1,11 @@ +#!/usr/bin/env python + import time from flask import Flask, render_template from flask_restful import Resource, Api import requests import psycopg2 +import json app = Flask(__name__) api = Api(app) @@ -12,36 +15,16 @@ DB_USER = "yggindex" DB_NAME = "yggindex" DB_HOST = "localhost" -#will add this as something seperate and pull down to a file rather -#than request it everytime. -def get_nodelist(): - data = requests.get("use the raw view of the github nodelist", timeout=1) - nodes = [x.split() for x in data.text.split('\n') if x] - - index_table = {} - - for key in nodes: - index_table[key[0]] = key[1] - return index_table - - -def check_nodelist(nodetable, key): - if nodetable: - if nodetable.get(key): - return nodetable.get(key) - else: - return key - else: - return key +ALIVE_SECONDS = 3600 # 1 hour def age_calc(ustamp): - if (time.time() - ustamp) <= 14400: + if (time.time() - ustamp) <= ALIVE_SECONDS: return True else: return False -#active nodes in the past 4hrs +# active nodes class nodesCurrent(Resource): def get(self): dbconn = psycopg2.connect(host=DB_HOST,\ @@ -65,63 +48,54 @@ class nodesCurrent(Resource): return nodelist -@app.route("/") -def fpage(): - dbconn = psycopg2.connect(host=DB_HOST,\ - database=DB_NAME,\ - user=DB_USER,\ - password=DB_PASSWORD) - cur = dbconn.cursor() - nodes = {} - cur.execute("select * from yggindex") - - for i in cur.fetchall(): - if age_calc(int(i[2])): - nodes[i[0]] = [i[1], int(i[2])] +# nodes info +class nodesInfo(Resource): + def get(self): + dbconn = psycopg2.connect(host=DB_HOST,\ + database=DB_NAME,\ + user=DB_USER,\ + password=DB_PASSWORD) + cur = dbconn.cursor() + nodes = {} + cur.execute("select * from yggnodeinfo") + for i in cur.fetchall(): + if age_calc(int(i[2])): + nodes[i[0]] = json.loads(i[1]) - dbconn.commit() - cur.close() - dbconn.close() + dbconn.commit() + cur.close() + dbconn.close() - return render_template('index.html', nodes=str(len(nodes))) + nodeinfo = {} + nodeinfo['yggnodeinfo'] = nodes + return nodeinfo -@app.route("/contrib") -def cpage(): - try: - domain_nodelist = get_nodelist() - print "list exists" - except: - print "failed" - domain_nodelist = None +@app.route("/") +def fpage(): dbconn = psycopg2.connect(host=DB_HOST,\ database=DB_NAME,\ user=DB_USER,\ password=DB_PASSWORD) cur = dbconn.cursor() - cur.execute("select * from contrib") - nodes = [] + nodes = 0 + cur.execute("select * from yggindex") for i in cur.fetchall(): - if age_calc(int(i[1])): - nodes.append(i[0]) + if age_calc(int(i[2])): + nodes += 1 dbconn.commit() cur.close() dbconn.close() - dnodes = [] - for key in nodes: - dnodes.append(check_nodelist(domain_nodelist, key)) - - dnodes.sort(reverse=True) - - return render_template('contrib.html', contribnodes=dnodes, nocontribs=str(len(dnodes))) + return render_template('index.html', nodes=nodes) #sort out the api request here for the url api.add_resource(nodesCurrent, '/current') +api.add_resource(nodesInfo, '/nodeinfo') if __name__ == '__main__': app.run(host='::', port=3000) diff --git a/api/static/style.css b/api/static/style.css index f202512..a50de86 100644 --- a/api/static/style.css +++ b/api/static/style.css @@ -1,28 +1,56 @@ /* style sheet for api pages */ body, input, textarea { - font-family: sans-serif; - padding: 0; - margin: 0; - background-color: #F0F0F0; -}a { - text-decoration:none; - color:#800000; -}h2 { - font-size:18px; -}#title { - max-width:570px; - margin-bottom:40px; - line-height: 35px; - font-size: 25px; - font-weight: bold; - padding-top:20px; - padding-bottom:5px; - border-bottom:1px solid #c4c4c4; -}#wrapper { - font-size: 15px; - max-width: 570px; - margin: auto; -}.contribs { - font-size:13px; - margin-bottom:6px; + font-family: sans-serif; + padding: 0; + margin: 0; + background-color: #F0F0F0; +} + +a { + text-decoration: none; + color: #800000; +} + +h2 { + font-size: 18px; +} + +#title { + max-width: 570px; + margin-bottom: 40px; + line-height: 35px; + font-size: 25px; + font-weight: bold; + padding-top: 20px; + padding-bottom: 5px; + border-bottom: 1px solid #c4c4c4; +} + +#title a { + color:#302f30; +} + +#wrapper { + font-size: 15px; + max-width: 570px; + margin: auto; +} + +.apireq { + padding-top: 5px; + padding-left: 10px; + padding-bottom: 5px; + line-height: 25px; + background-color: #e5e5e5; + color: #4c4c4c; + border-top: solid 1px #d3d3d3; + border-bottom: solid 1px #d3d3d3; + margin-top: 8px; + margin-bottom: 20px; +} + +.wide { + width: 100%; + border-bottom: solid 1px #d3d3d3; + margin-bottom: 25px; } diff --git a/api/templates/contrib.html b/api/templates/contrib.html deleted file mode 100644 index 2e16d8d..0000000 --- a/api/templates/contrib.html +++ /dev/null @@ -1,21 +0,0 @@ - -API - - - -
-
- Yggdrasil Network API -
- -

Contributors - {{nocontribs}}

-

This is a list of contributors sending in their view of the network, this list is based on the contributions sent in the last 4 hours.


- - {% for x in contribnodes %} -
{{ x }}
- {% endfor %} - - -
- - diff --git a/api/templates/index.html b/api/templates/index.html index 73bf603..2857861 100644 --- a/api/templates/index.html +++ b/api/templates/index.html @@ -2,22 +2,36 @@ API - -
+ +
- This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.


+ This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.

- Current Nodes online: {{ nodes }}

+
+
+ Current Nodes online
+ {{ nodes }} +
+

+ Yggdrasil Interactive World map

- Yggdrasil World map

- Contributors


- Make an API request:

- http://y.yakamo.org:3000/current


- To contribute to the yggdrasil world map, you can send your view of the network using send-view.py. Set a Cron job to once an hour.

+
+ Make an API request
+ note: data updated every 15 minutes, all requests are returned in JSON.

- Thanks to Arceliar for the map generating script. -
- + Get a current list of active and online nodes:
+
+ http://[31a:fb8a:c43e:ca59::2]/current +
+ Nodeino from all current active nodes:
+
+ http://[31a:fb8a:c43e:ca59::2]/nodeinfo +
+ +
+ Made with Niflheim-API by yakamok +
+ diff --git a/api/yggapi.service b/api/yggapi.service index b460811..57a1a4b 100644 --- a/api/yggapi.service +++ b/api/yggapi.service @@ -3,9 +3,11 @@ Description=yggapi After=network.target [Service] -User=yakamo -ExecStart=/usr/bin/python /opt/yggapi.py +WorkingDirectory=/opt/yggdrasil-api/api +ExecStart=/opt/yggdrasil-api/venv/bin/python /opt/yggdrasil-api/api/niflheim-api.py Restart=always +EnvironmentFile=/opt/yggdrasil-api/venv/bin/activate + [Install] WantedBy=multi-user.target diff --git a/requirements.txt b/requirements.txt new file mode 100644 index 0000000..f4642cc --- /dev/null +++ b/requirements.txt @@ -0,0 +1,4 @@ +flask +flask_restful +psycopg2 +requests