correction updates, make it usable

api/crawler.py

@@ -1,6 +1,8 @@
-#some of this code was contributed by Arcelier
-#original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
-#multithreaded by neilalexander
+#!/usr/bin/env python
+
+# some of this code was contributed by Arcelier
+# original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
+# multithreaded by neilalexander
 
 import psycopg2
 import json
@@ -12,6 +14,20 @@ import traceback
 from threading import Lock, Thread
 from Queue import Queue
 
+# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
+useAdminSock = True
+yggAdminTCP  = ('localhost', 9001)
+yggAdminSock = ('/var/run/yggdrasil.sock')
+
+DB_PASSWORD = "password"
+DB_USER = "yggindex"
+DB_NAME = "yggindex"
+DB_HOST = "localhost"
+
+## Save in database node info fields like buildname, buildarch, buildplatform, buildversion (True/False)?
+saveDefaultNodeInfo = False
+removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion']
+
 class Worker(Thread):
     def __init__(self, tasks):
         Thread.__init__(self)
@@ -48,13 +64,6 @@ nodeinfo = dict()
 nodeinfomutex = Lock()
 nodeinfopool = ThreadPool(30)
 
-host_port = ('localhost', 9001)
-
-DB_PASSWORD = "akemi2501"
-DB_USER = "yggindex"
-DB_NAME = "yggindex"
-DB_HOST = "localhost"
-
 def recv_until_done(soc):
     all_data = []
     while True:
@@ -83,8 +92,13 @@ def getNodeInfoTask(address, info):
 
 def doRequest(req):
     try:
-        ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
-        ygg.connect(host_port)
+        if useAdminSock:
+            ygg = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
+            ygg.connect(yggAdminSock)
+        else:
+            ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
+            ygg.connect(yggAdminTCP)
+
         ygg.send(req)
         data = json.loads(recv_until_done(ygg))
         return data
@@ -118,24 +132,34 @@ def insert_new_entry(ipv6, coords):
         nodejson = "{}"
         if ipv6 in nodeinfo:
             with nodeinfomutex:
+
+                if not saveDefaultNodeInfo:
+                    # remove default Node info fields
+                    for field in removableFileds:
+                        tmprm = nodeinfo[ipv6].pop(field, None)
+
                 nodejson = json.dumps(nodeinfo[ipv6])
                 nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
+
         dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
         cur = dbconn.cursor()
+        timestamp = str(int(time.time()))
+
         cur.execute(
             "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
-            (ipv6, coords, str(int(time.time())), nodename, str(int(time.time())), coords, nodename)
+            (ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
         )
         cur.execute(
-            "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, NOW()) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=NOW();",
-            (ipv6, nodejson, nodejson)
+            "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
+            (ipv6, nodejson, timestamp, nodejson, timestamp)
         )
+
         dbconn.commit()
         cur.close()
         dbconn.close()
     except Exception as e:
-        print("database error inserting")
-	traceback.print_exc()
+        print "database error inserting"
+        traceback.print_exc()
 
 def handleNodeInfo(address, data):
     global nodeinfo
@@ -183,6 +207,7 @@ for k,v in selfInfo['response']['self'].iteritems():
 # Loop over rumored nodes and ping them, adding to visited if they respond
 while len(rumored) > 0:
     for k,v in rumored.iteritems():
+        #print "Processing", v['coords']
         handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'])))
         break
     del rumored[k]

api/max-min.py

@@ -1,4 +1,7 @@
+#!/usr/bin/env python
+
 #max/min for the day with nodes
+
 import psycopg2
 import time
 

api/niflheim-api.py

@@ -1,8 +1,11 @@
+#!/usr/bin/env python
+
 import time
 from flask import Flask, render_template
 from flask_restful import Resource, Api
 import requests
 import psycopg2
+import json
 
 app = Flask(__name__)
 api = Api(app)
@@ -12,36 +15,16 @@ DB_USER = "yggindex"
 DB_NAME = "yggindex"
 DB_HOST = "localhost"
 
-#will add this as something seperate and pull down to a file rather
-#than request it everytime.
-def get_nodelist():
-    data = requests.get("use the raw view of the github nodelist", timeout=1)
-    nodes = [x.split() for x in data.text.split('\n') if x]
-
-    index_table = {}
-
-    for key in nodes:
-        index_table[key[0]] = key[1]
-    return index_table
-
-
-def check_nodelist(nodetable, key):
-    if nodetable:
-        if nodetable.get(key):
-            return nodetable.get(key)
-        else:
-            return key
-    else:
-        return key
+ALIVE_SECONDS = 3600 # 1 hour
 
 
 def age_calc(ustamp):
-    if (time.time() - ustamp) <= 14400:
+    if (time.time() - ustamp) <= ALIVE_SECONDS:
         return True
     else:
         return False
 
-#active nodes in the past 4hrs
+# active nodes
 class nodesCurrent(Resource):
     def get(self):
         dbconn = psycopg2.connect(host=DB_HOST,\
@@ -65,63 +48,54 @@ class nodesCurrent(Resource):
         return nodelist
 
 
-@app.route("/")
-def fpage():
-    dbconn = psycopg2.connect(host=DB_HOST,\
-                                database=DB_NAME,\
-                                user=DB_USER,\
-                                password=DB_PASSWORD)
-    cur = dbconn.cursor()
-    nodes = {}
-    cur.execute("select * from yggindex")
-
-    for i in cur.fetchall():
-        if age_calc(int(i[2])):
-            nodes[i[0]] = [i[1], int(i[2])]
+# nodes info
+class nodesInfo(Resource):
+    def get(self):
+        dbconn = psycopg2.connect(host=DB_HOST,\
+                                    database=DB_NAME,\
+                                    user=DB_USER,\
+                                    password=DB_PASSWORD)
+        cur = dbconn.cursor()
+        nodes = {}
+        cur.execute("select * from yggnodeinfo")
+        for i in cur.fetchall():
+            if age_calc(int(i[2])):
+                nodes[i[0]] = json.loads(i[1])
 
-    dbconn.commit()
-    cur.close()
-    dbconn.close()
+        dbconn.commit()
+        cur.close()
+        dbconn.close()
 
-    return render_template('index.html', nodes=str(len(nodes)))
+        nodeinfo = {}
+        nodeinfo['yggnodeinfo'] = nodes
 
+        return nodeinfo
 
-@app.route("/contrib")
-def cpage():
-    try:
-        domain_nodelist = get_nodelist()
-        print "list exists"
-    except:
-        print "failed"
-        domain_nodelist = None
 
+@app.route("/")
+def fpage():
     dbconn = psycopg2.connect(host=DB_HOST,\
                                 database=DB_NAME,\
                                 user=DB_USER,\
                                 password=DB_PASSWORD)
     cur = dbconn.cursor()
-    cur.execute("select * from contrib")
-    nodes = []
+    nodes = 0
+    cur.execute("select * from yggindex")
 
     for i in cur.fetchall():
-        if age_calc(int(i[1])):
-            nodes.append(i[0])
+        if age_calc(int(i[2])):
+            nodes += 1
 
     dbconn.commit()
     cur.close()
     dbconn.close()
 
-    dnodes = []
-    for key in nodes:
-        dnodes.append(check_nodelist(domain_nodelist, key))
-
-    dnodes.sort(reverse=True)
-
-    return render_template('contrib.html', contribnodes=dnodes, nocontribs=str(len(dnodes)))
+    return render_template('index.html', nodes=nodes)
 
 
 #sort out the api request here for the url
 api.add_resource(nodesCurrent, '/current')
+api.add_resource(nodesInfo, '/nodeinfo')
 
 if __name__ == '__main__':
     app.run(host='::', port=3000)

api/static/style.css

@@ -1,28 +1,56 @@
 /* style sheet for api pages */
 body, input, textarea {
- font-family: sans-serif;
- padding: 0;
- margin: 0;
- background-color: #F0F0F0;
-}a {
- text-decoration:none;
- color:#800000;
-}h2 {
- font-size:18px;
-}#title {
- max-width:570px;
- margin-bottom:40px;
- line-height: 35px;
- font-size: 25px;
- font-weight: bold;
- padding-top:20px;
- padding-bottom:5px;
- border-bottom:1px solid #c4c4c4;
-}#wrapper {
- font-size: 15px;
- max-width: 570px;
- margin: auto;
-}.contribs {
- font-size:13px;
- margin-bottom:6px;
+  font-family: sans-serif;
+  padding: 0;
+  margin: 0;
+  background-color: #F0F0F0;
+}
+
+a {
+  text-decoration: none;
+  color: #800000;
+}
+
+h2 {
+  font-size: 18px;
+}
+
+#title {
+	max-width: 570px;
+	margin-bottom: 40px;
+	line-height: 35px;
+	font-size: 25px;
+	font-weight: bold;
+	padding-top: 20px;
+	padding-bottom: 5px;
+	border-bottom: 1px solid #c4c4c4;
+}
+
+#title a {
+	color:#302f30;
+}
+
+#wrapper {
+	font-size: 15px;
+	max-width: 570px;
+	margin: auto;
+}
+
+.apireq {
+	padding-top: 5px;
+	padding-left: 10px;
+	padding-bottom: 5px;
+	line-height: 25px;
+	background-color: #e5e5e5;
+	color: #4c4c4c;
+	border-top: solid 1px #d3d3d3;
+	border-bottom: solid 1px #d3d3d3;
+	margin-top: 8px;
+	margin-bottom: 20px;
+}
+
+.wide {
+	width: 100%;
+	border-bottom: solid 1px #d3d3d3;
+	margin-bottom: 25px;
 }

api/templates/contrib.html

@@ -1,21 +0,0 @@
-<!doctype html>
-<title>API</title>
-<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
-<html>
-   <body>
-    <div id="wrapper">
-            <div id="title">
-		    <a href='/'>Yggdrasil Network API</a>
-            </div>
-
-	    <h2>Contributors - {{nocontribs}}</h2>
-	    <p>This is a list of contributors sending in their view of the network, this list is based on the contributions sent in the last 4 hours.</p><br />
-
-	  {% for x in contribnodes %}
-		  <div class="contribs">{{ x }}</div>
-	  {% endfor %}
-
-
-    </div>
-   </body>
-</html>

api/templates/index.html

@@ -2,22 +2,36 @@
 <title>API</title>
 <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
 <html>
-   <body>
-    <div id="wrapper">
+    <body>
+        <div id="wrapper">
             <div id="title">
-	    <a href='/'>Yggdrasil Network API</a>
+                <a href='/'>Yggdrasil Network API</a>
             </div>
-            This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br /><br />
+            This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br />
 
-            <strong>Current Nodes online:</strong> {{ nodes }}<br /><br />
+            <div class="wide"></div>
+            <center>
+                <strong>Current Nodes online<br />
+                <font size="18">{{ nodes }}</font></strong>
+            </center>
+            <br /><br />
+            <a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
 
-            <a href="static/map.svg">Yggdrasil World map</a><br/><br/> 
-	    <a href="/contrib">Contributors</a><br /><br /><br />
-            <strong>Make an API request:</strong><br/><br/>
-            <a href="/current">http://y.yakamo.org:3000/current</a><br/><br/><br/>
-            To contribute to the yggdrasil world map, you can send your view of the network using <a href="https://github.com/yakamok/ygg-node-db/blob/master/send-view.py">send-view.py</a>. Set a Cron job to once an hour.<br/><br/>
+            <div class="wide"></div>
+            <strong>Make an API request</strong><br />
+            <small>note: data updated every 15 minutes, all requests are returned in JSON.</small><br /><br />
 
-            Thanks to Arceliar for the map generating script.
-    </div>
-   </body>
+            Get a current list of active and online nodes:<br />
+            <div class="apireq">
+                <a href="/current">http://[31a:fb8a:c43e:ca59::2]/current</a>
+            </div>
+            Nodeino from all current active nodes:<br />
+            <div class="apireq">
+                <a href="/nodeinfo">http://[31a:fb8a:c43e:ca59::2]/nodeinfo</a>
+            </div>
+
+            <div class="wide"></div>
+            <small>Made with <a href="https://github.com/yakamok/Niflheim-api">Niflheim-API</a> by yakamok</small>
+        </div>
+    </body>
 </html>

api/yggapi.service

@@ -3,9 +3,11 @@ Description=yggapi
 After=network.target
 
 [Service]
-User=yakamo
-ExecStart=/usr/bin/python /opt/yggapi.py
+WorkingDirectory=/opt/yggdrasil-api/api
+ExecStart=/opt/yggdrasil-api/venv/bin/python /opt/yggdrasil-api/api/niflheim-api.py
 Restart=always
 
+EnvironmentFile=/opt/yggdrasil-api/venv/bin/activate
+
 [Install]
 WantedBy=multi-user.target

requirements.txt

@@ -0,0 +1,4 @@
+flask
+flask_restful
+psycopg2
+requests