add importer from yggcrawl, update template

.gitignore

@@ -0,0 +1,3 @@
+/venv
+results.json
+yggcrawl.log

api/crawler.py

@@ -13,12 +13,14 @@ import traceback
 from threading import Lock, Thread
 from queue import Queue
 
+#####
+
 # Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
 useAdminSock = True
 yggAdminTCP  = ('localhost', 9001)
 yggAdminSock = ('/var/run/yggdrasil.sock')
 
-DB_PASSWORD = "password"
+DB_PASS = "password"
 DB_USER = "yggindex"
 DB_NAME = "yggindex"
 DB_HOST = "localhost"
@@ -27,6 +29,8 @@ DB_HOST = "localhost"
 saveDefaultNodeInfo = False
 removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
 
+#####
+
 class Worker(Thread):
     def __init__(self, tasks):
         Thread.__init__(self)
@@ -140,7 +144,7 @@ def insert_new_entry(ipv6, coords):
                 nodejson = json.dumps(nodeinfo[ipv6])
                 nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
 
-        dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
+        dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
         cur = dbconn.cursor()
         timestamp = str(int(time.time()))
 

api/importer.py

@@ -0,0 +1,69 @@
+#!/usr/bin/env python
+
+import psycopg2, json, traceback
+
+#####
+
+# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
+DB_PASS = "password"
+DB_USER = "yggindex"
+DB_NAME = "yggindex"
+DB_HOST = "localhost"
+
+## Save in database node info fields like buildname, buildarch, etc. (True/False)?
+saveDefaultNodeInfo = False
+removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion', 'board_name', 'kernel', 'model', 'system']
+
+#####
+
+with open('api/results.json', 'r') as f:
+    data = json.load(f)
+
+timestamp = data['meta']['generated_at_utc']
+
+# connect to database
+dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASS)
+cur = dbconn.cursor()
+
+# start importing
+for node in data['topology']:
+    nodename = ""
+    nodeinfo = {}
+    ipv6 = data['topology'][node]['ipv6_addr']
+    coords = '[%s]' % (' '.join(str(e) for e in data['topology'][node]['coords']))
+
+    if node in data['nodeinfo']:
+        nodeinfo = data['nodeinfo'][node]
+
+        if not saveDefaultNodeInfo:
+            # remove default Node info fields
+            for field in removableFileds:
+                tmprm = nodeinfo.pop(field, None)
+
+    if "name" in nodeinfo:
+        nodename = nodeinfo['name']
+    elif data['topology'][node]['found'] == False:
+        nodename = '? %s' % coords
+    else:
+        nodename = ipv6
+
+    nodeinfo = json.dumps(nodeinfo)
+
+    try:
+        cur.execute(
+            "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
+            (ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
+        )
+        cur.execute(
+            "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
+            (ipv6, nodeinfo, timestamp, nodeinfo, timestamp)
+        )
+
+    except Exception as e:
+        print("database error inserting")
+        traceback.print_exc()
+
+dbconn.commit()
+cur.close()
+dbconn.close()
+

api/max-min.py

@@ -7,7 +7,7 @@ import time
 
 #run every hour
 
-DB_PASSWORD = "password"
+DB_PASS = "password"
 DB_USER = "yggindex"
 DB_NAME = "yggindex"
 DB_HOST = "localhost"
@@ -24,7 +24,7 @@ def age_calc(ustamp):
 		return False
 
 def get_nodes_for_count():
-	dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
+	dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
 	cur = dbconn.cursor()
 	nodes = {}
 	cur.execute("select * from yggindex")
@@ -39,7 +39,7 @@ def get_nodes_for_count():
 	return str(len(nodes))
 
 def add_to_db():
-	dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
+	dbconn = psycopg2.connect(host=DB_HOST,database=DB_NAME, user=DB_USER, password=DB_PASS)
 	cur = dbconn.cursor()
 
 	cur.execute('''INSERT INTO timeseries(max, unixtstamp) VALUES(''' + "'" + get_nodes_for_count() + "'," + str(int(time.time())) + ''')''')

api/niflheim-api.py

@@ -1,16 +1,15 @@
 #!/usr/bin/env python
 
-import time
+import signal, sys, time
 from flask import Flask, render_template
 from flask_restful import Resource, Api
 import requests
 import psycopg2
 import json
 
-app = Flask(__name__)
+######
-api = Api(app)
 
-DB_PASSWORD = "password"
+DB_PASS = "password"
 DB_USER = "yggindex"
 DB_NAME = "yggindex"
 DB_HOST = "localhost"
@@ -19,6 +18,19 @@ DB_HOST = "localhost"
 # I'm using 1 hour beause of running crawler every 15 minutes
 ALIVE_SECONDS = 3600 # 1 hour
 
+######
+
+app = Flask(__name__)
+api = Api(app)
+
+dbconn = psycopg2.connect(host=DB_HOST,\
+                          database=DB_NAME,\
+                          user=DB_USER,\
+                          password=DB_PASS)
+
+def signal_handler(sig, frame):
+    dbconn.close()
+    sys.exit(0)
 
 def age_calc(ustamp):
     if (time.time() - ustamp) <= ALIVE_SECONDS:
@@ -29,10 +41,6 @@ def age_calc(ustamp):
 # active nodes
 class nodesCurrent(Resource):
     def get(self):
-        dbconn = psycopg2.connect(host=DB_HOST,\
-                                    database=DB_NAME,\
-                                    user=DB_USER,\
-                                    password=DB_PASSWORD)
         cur = dbconn.cursor()
         nodes = {}
         cur.execute("select * from yggindex")
@@ -42,7 +50,6 @@ class nodesCurrent(Resource):
 
         dbconn.commit()
         cur.close()
-        dbconn.close()
 
         nodelist = {}
         nodelist['yggnodes'] = nodes
@@ -53,10 +60,6 @@ class nodesCurrent(Resource):
 # nodes info
 class nodesInfo(Resource):
     def get(self):
-        dbconn = psycopg2.connect(host=DB_HOST,\
-                                    database=DB_NAME,\
-                                    user=DB_USER,\
-                                    password=DB_PASSWORD)
         cur = dbconn.cursor()
         nodes = {}
         cur.execute("select * from yggnodeinfo")
@@ -66,7 +69,6 @@ class nodesInfo(Resource):
 
         dbconn.commit()
         cur.close()
-        dbconn.close()
 
         nodeinfo = {}
         nodeinfo['yggnodeinfo'] = nodes
@@ -77,10 +79,6 @@ class nodesInfo(Resource):
 # alive nodes count for latest 24 hours
 class nodes24h(Resource):
     def get(self):
-        dbconn = psycopg2.connect(host=DB_HOST,\
-                                    database=DB_NAME,\
-                                    user=DB_USER,\
-                                    password=DB_PASSWORD)
         cur = dbconn.cursor()
         nodes = {}
         cur.execute("SELECT * FROM timeseries ORDER BY unixtstamp DESC LIMIT 24")
@@ -89,21 +87,23 @@ class nodes24h(Resource):
 
         dbconn.commit()
         cur.close()
-        dbconn.close()
 
         nodeinfo = {}
         nodeinfo['nodes24h'] = nodes
 
         return nodeinfo
 
+# alive nodes count for latest 24 hours
+class crawlResult(Resource):
+    def get(self):
+        with open('api/results.json', 'r') as f:
+            data = json.load(f)
+
+        return data
 
 
 @app.route("/")
 def fpage():
-    dbconn = psycopg2.connect(host=DB_HOST,\
-                                database=DB_NAME,\
-                                user=DB_USER,\
-                                password=DB_PASSWORD)
     cur = dbconn.cursor()
     nodes = 0
     cur.execute("select * from yggindex")
@@ -114,15 +114,19 @@ def fpage():
 
     dbconn.commit()
     cur.close()
-    dbconn.close()
 
     return render_template('index.html', nodes=nodes)
 
 
-#sort out the api request here for the url
+# sort out the api request here for the url
 api.add_resource(nodesCurrent, '/current')
 api.add_resource(nodesInfo, '/nodeinfo')
 api.add_resource(nodes24h, '/nodes24h')
+api.add_resource(crawlResult, '/result.json')
+
+# regirster signal handler
+signal.signal(signal.SIGINT, signal_handler)
+signal.signal(signal.SIGTERM, signal_handler)
 
 if __name__ == '__main__':
     app.run(host='::', port=3000)

api/templates/index.html

@@ -13,9 +13,10 @@
             <center>
                 <strong>Current Nodes online<br />
                 <font size="18">{{ nodes }}</font></strong>
+                <br /><br />
+                You can see them in <i>Yggdrasil Interactive World map</i> <sup><a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">[&#8663;]</a> <a href="http://map.ygg/" target="_blank">[DNS]</a></sup>
             </center>
             <br /><br />
-            <a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
 
             <div class="wide"></div>
             <strong>Make an API request</strong><br />
@@ -23,17 +24,22 @@
 
             Get a current list of active and online nodes:<br />
             <div class="apireq">
-                http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="/current" target="_blank">&#8663;</a></sup>
+                http://[31a:fb8a:c43e:ca59::2]/current <sup><a href="http://[31a:fb8a:c43e:ca59::2]/current" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/current" target="_blank">[DNS]</a></sup>
             </div>
             Nodeinfo from all current active nodes:<br />
             <div class="apireq">
-                http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="/nodeinfo" target="_blank">&#8663;</a></sup>
+                http://[31a:fb8a:c43e:ca59::2]/nodeinfo <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodeinfo" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodeinfo" target="_blank">[DNS]</a></sup>
             </div>
             Active nodes count for last 24 hours:<br />
             <div class="apireq">
-                http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="/nodes24h" target="_blank">&#8663;</a></sup>
+                http://[31a:fb8a:c43e:ca59::2]/nodes24h <sup><a href="http://[31a:fb8a:c43e:ca59::2]/nodes24h" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/nodes24h" target="_blank">[DNS]</a></sup>
             </div>
 
+            <div class="wide"></div>
+            <strong>Get latest crawler data</strong><br />
+            You can download data in raw json provided by <a href="https://github.com/neilalexander/yggcrawl" target="_blank">yggcrawl</a>: result.json <sup><a href="http://[31a:fb8a:c43e:ca59::2]/result.json" target="_blank">[&#8663;]</a> <a href="http://nodelist.ygg/result.json" target="_blank">[DNS]</a></sup>
+            <br /><br />
+
             <div class="wide"></div>
             <small>Made with <a href="https://github.com/r4sas/Niflheim-api" target="_blank">fork</a> of <a href="https://github.com/yakamok/Niflheim-api" target="_blank">Niflheim-API</a> by yakamok</small>
         </div>

ygg-crawl.sh

@@ -0,0 +1,12 @@
+#!/bin/sh
+
+YGGCRAWL="/opt/yggcrawl/yggcrawl" # path to yggcrawl binary
+YGGAPIPATH="/opt/yggdrasil-api"   # path to Niflheim-API directory
+
+CRAWLPEER="tcp://127.0.0.1:12345" # Yggdrasil peer address
+CRAWLFILE="api/results.json"
+CRAWLRETR=3
+
+cd $YGGAPIPATH
+$YGGCRAWL -peer $CRAWLPEER -retry $CRAWLRETR -file $CRAWLFILE > api/yggcrawl.log 2>&1
+venv/bin/python api/importer.py >> api/yggcrawl.log 2>&1