r4sas
/
yggdrasil-api
mirror of https://github.com/r4sas/Niflheim-api
1
0
Fork 0
Code Issues Releases Wiki Activity
Browse Source

correction updates, make it usable

master
R4SAS 5 years ago
parent
d9a062362b
commit
ce51013715
8 changed files with 166 additions and 137 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 59
      api/crawler.py
  2. 3
      api/max-min.py
  3. 92
      api/niflheim-api.py
  4. 78
      api/static/style.css
  5. 21
      api/templates/contrib.html
  6. 40
      api/templates/index.html
  7. 6
      api/yggapi.service
  8. 4
      requirements.txt

59
api/crawler.py
Unescape View File

@ -1,6 +1,8 @@
#some of this code was contributed by Arcelier #!/usr/bin/env python
#original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
#multithreaded by neilalexander # some of this code was contributed by Arcelier
# original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
# multithreaded by neilalexander
import psycopg2 import psycopg2
import json import json
@ -12,6 +14,20 @@ import traceback
from threading import Lock, Thread from threading import Lock, Thread
from Queue import Queue from Queue import Queue
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
useAdminSock = True
yggAdminTCP = ('localhost', 9001)
yggAdminSock = ('/var/run/yggdrasil.sock')
DB_PASSWORD = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
## Save in database node info fields like buildname, buildarch, buildplatform, buildversion (True/False)?
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion']
class Worker(Thread): class Worker(Thread):
def __init__(self, tasks): def __init__(self, tasks):
Thread.__init__(self) Thread.__init__(self)
@ -48,13 +64,6 @@ nodeinfo = dict()
nodeinfomutex = Lock() nodeinfomutex = Lock()
nodeinfopool = ThreadPool(30) nodeinfopool = ThreadPool(30)
host_port = ('localhost', 9001)
DB_PASSWORD = "akemi2501"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
def recv_until_done(soc): def recv_until_done(soc):
all_data = [] all_data = []
while True: while True:
@ -83,8 +92,13 @@ def getNodeInfoTask(address, info):
def doRequest(req): def doRequest(req):
try: try:
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if useAdminSock:
ygg.connect(host_port) ygg = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
ygg.connect(yggAdminSock)
else:
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ygg.connect(yggAdminTCP)
ygg.send(req) ygg.send(req)
data = json.loads(recv_until_done(ygg)) data = json.loads(recv_until_done(ygg))
return data return data
@ -118,24 +132,34 @@ def insert_new_entry(ipv6, coords):
nodejson = "{}" nodejson = "{}"
if ipv6 in nodeinfo: if ipv6 in nodeinfo:
with nodeinfomutex: with nodeinfomutex:
if not saveDefaultNodeInfo:
# remove default Node info fields
for field in removableFileds:
tmprm = nodeinfo[ipv6].pop(field, None)
nodejson = json.dumps(nodeinfo[ipv6]) nodejson = json.dumps(nodeinfo[ipv6])
nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else "" nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD) dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
cur = dbconn.cursor() cur = dbconn.cursor()
timestamp = str(int(time.time()))
cur.execute( cur.execute(
"INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;", "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
(ipv6, coords, str(int(time.time())), nodename, str(int(time.time())), coords, nodename) (ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
) )
cur.execute( cur.execute(
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, NOW()) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=NOW();", "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
(ipv6, nodejson, nodejson) (ipv6, nodejson, timestamp, nodejson, timestamp)
) )
dbconn.commit() dbconn.commit()
cur.close() cur.close()
dbconn.close() dbconn.close()
except Exception as e: except Exception as e:
print("database error inserting") print "database error inserting"
traceback.print_exc() traceback.print_exc()
def handleNodeInfo(address, data): def handleNodeInfo(address, data):
global nodeinfo global nodeinfo
@ -183,6 +207,7 @@ for k,v in selfInfo['response']['self'].iteritems():
# Loop over rumored nodes and ping them, adding to visited if they respond # Loop over rumored nodes and ping them, adding to visited if they respond
while len(rumored) > 0: while len(rumored) > 0:
for k,v in rumored.iteritems(): for k,v in rumored.iteritems():
#print "Processing", v['coords']
handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords']))) handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'])))
break break
del rumored[k] del rumored[k]

3
api/max-min.py
Unescape View File

@ -1,4 +1,7 @@
#!/usr/bin/env python
#max/min for the day with nodes #max/min for the day with nodes
import psycopg2 import psycopg2
import time import time

92
api/niflheim-api.py
Unescape View File

@ -1,8 +1,11 @@
#!/usr/bin/env python
import time import time
from flask import Flask, render_template from flask import Flask, render_template
from flask_restful import Resource, Api from flask_restful import Resource, Api
import requests import requests
import psycopg2 import psycopg2
import json
app = Flask(__name__) app = Flask(__name__)
api = Api(app) api = Api(app)
@ -12,36 +15,16 @@ DB_USER = "yggindex"
DB_NAME = "yggindex" DB_NAME = "yggindex"
DB_HOST = "localhost" DB_HOST = "localhost"
#will add this as something seperate and pull down to a file rather ALIVE_SECONDS = 3600 # 1 hour
#than request it everytime.
def get_nodelist():
data = requests.get("use the raw view of the github nodelist", timeout=1)
nodes = [x.split() for x in data.text.split('\n') if x]
index_table = {}
for key in nodes:
index_table[key[0]] = key[1]
return index_table
def check_nodelist(nodetable, key):
if nodetable:
if nodetable.get(key):
return nodetable.get(key)
else:
return key
else:
return key
def age_calc(ustamp): def age_calc(ustamp):
if (time.time() - ustamp) <= 14400: if (time.time() - ustamp) <= ALIVE_SECONDS:
return True return True
else: else:
return False return False
#active nodes in the past 4hrs # active nodes
class nodesCurrent(Resource): class nodesCurrent(Resource):
def get(self): def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\ dbconn = psycopg2.connect(host=DB_HOST,\
@ -65,63 +48,54 @@ class nodesCurrent(Resource):
return nodelist return nodelist
@app.route("/") # nodes info
def fpage(): class nodesInfo(Resource):
dbconn = psycopg2.connect(host=DB_HOST,\ def get(self):
database=DB_NAME,\ dbconn = psycopg2.connect(host=DB_HOST,\
user=DB_USER,\ database=DB_NAME,\
password=DB_PASSWORD) user=DB_USER,\
cur = dbconn.cursor() password=DB_PASSWORD)
nodes = {} cur = dbconn.cursor()
cur.execute("select * from yggindex") nodes = {}
cur.execute("select * from yggnodeinfo")
for i in cur.fetchall(): for i in cur.fetchall():
if age_calc(int(i[2])): if age_calc(int(i[2])):
nodes[i[0]] = [i[1], int(i[2])] nodes[i[0]] = json.loads(i[1])
dbconn.commit() dbconn.commit()
cur.close() cur.close()
dbconn.close() dbconn.close()
return render_template('index.html', nodes=str(len(nodes))) nodeinfo = {}
nodeinfo['yggnodeinfo'] = nodes
return nodeinfo
@app.route("/contrib")
def cpage():
try:
domain_nodelist = get_nodelist()
print "list exists"
except:
print "failed"
domain_nodelist = None
@app.route("/")
def fpage():
dbconn = psycopg2.connect(host=DB_HOST,\ dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\ database=DB_NAME,\
user=DB_USER,\ user=DB_USER,\
password=DB_PASSWORD) password=DB_PASSWORD)
cur = dbconn.cursor() cur = dbconn.cursor()
cur.execute("select * from contrib") nodes = 0
nodes = [] cur.execute("select * from yggindex")
for i in cur.fetchall(): for i in cur.fetchall():
if age_calc(int(i[1])): if age_calc(int(i[2])):
nodes.append(i[0]) nodes += 1
dbconn.commit() dbconn.commit()
cur.close() cur.close()
dbconn.close() dbconn.close()
dnodes = [] return render_template('index.html', nodes=nodes)
for key in nodes:
dnodes.append(check_nodelist(domain_nodelist, key))
dnodes.sort(reverse=True)
return render_template('contrib.html', contribnodes=dnodes, nocontribs=str(len(dnodes)))
#sort out the api request here for the url #sort out the api request here for the url
api.add_resource(nodesCurrent, '/current') api.add_resource(nodesCurrent, '/current')
api.add_resource(nodesInfo, '/nodeinfo')
if __name__ == '__main__': if __name__ == '__main__':
app.run(host='::', port=3000) app.run(host='::', port=3000)

78
api/static/style.css
Unescape View File

@ -1,28 +1,56 @@
/* style sheet for api pages */ /* style sheet for api pages */
body, input, textarea { body, input, textarea {
font-family: sans-serif; font-family: sans-serif;
padding: 0; padding: 0;
margin: 0; margin: 0;
background-color: #F0F0F0; background-color: #F0F0F0;
}a { }
text-decoration:none;
color:#800000; a {
}h2 { text-decoration: none;
font-size:18px; color: #800000;
}#title { }
max-width:570px;
margin-bottom:40px; h2 {
line-height: 35px; font-size: 18px;
font-size: 25px; }
font-weight: bold;
padding-top:20px; #title {
padding-bottom:5px; max-width: 570px;
border-bottom:1px solid #c4c4c4; margin-bottom: 40px;
}#wrapper { line-height: 35px;
font-size: 15px; font-size: 25px;
max-width: 570px; font-weight: bold;
margin: auto; padding-top: 20px;
}.contribs { padding-bottom: 5px;
font-size:13px; border-bottom: 1px solid #c4c4c4;
margin-bottom:6px; }
#title a {
color:#302f30;
}
#wrapper {
font-size: 15px;
max-width: 570px;
margin: auto;
}
.apireq {
padding-top: 5px;
padding-left: 10px;
padding-bottom: 5px;
line-height: 25px;
background-color: #e5e5e5;
color: #4c4c4c;
border-top: solid 1px #d3d3d3;
border-bottom: solid 1px #d3d3d3;
margin-top: 8px;
margin-bottom: 20px;
}
.wide {
width: 100%;
border-bottom: solid 1px #d3d3d3;
margin-bottom: 25px;
} }

21
api/templates/contrib.html
Unescape View File

@ -1,21 +0,0 @@
<!doctype html>
<title>API</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
<html>
<body>
<div id="wrapper">
<div id="title">
<a href='/'>Yggdrasil Network API</a>
</div>
<h2>Contributors - {{nocontribs}}</h2>
<p>This is a list of contributors sending in their view of the network, this list is based on the contributions sent in the last 4 hours.</p><br />
{% for x in contribnodes %}
<div class="contribs">{{ x }}</div>
{% endfor %}
</div>
</body>
</html>

40
api/templates/index.html
Unescape View File

@ -2,22 +2,36 @@
<title>API</title> <title>API</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/> <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
<html> <html>
<body> <body>
<div id="wrapper"> <div id="wrapper">
<div id="title"> <div id="title">
<a href='/'>Yggdrasil Network API</a> <a href='/'>Yggdrasil Network API</a>
</div> </div>
This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br /><br /> This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br />
<strong>Current Nodes online:</strong> {{ nodes }}<br /><br /> <div class="wide"></div>
<center>
<strong>Current Nodes online<br />
<font size="18">{{ nodes }}</font></strong>
</center>
<br /><br />
<a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
<a href="static/map.svg">Yggdrasil World map</a><br/><br/> <div class="wide"></div>
<a href="/contrib">Contributors</a><br /><br /><br /> <strong>Make an API request</strong><br />
<strong>Make an API request:</strong><br/><br/> <small>note: data updated every 15 minutes, all requests are returned in JSON.</small><br /><br />
<a href="/current">http://y.yakamo.org:3000/current</a><br/><br/><br/>
To contribute to the yggdrasil world map, you can send your view of the network using <a href="https://github.com/yakamok/ygg-node-db/blob/master/send-view.py">send-view.py</a>. Set a Cron job to once an hour.<br/><br/>
Thanks to Arceliar for the map generating script. Get a current list of active and online nodes:<br />
</div> <div class="apireq">
</body> <a href="/current">http://[31a:fb8a:c43e:ca59::2]/current</a>
</div>
Nodeino from all current active nodes:<br />
<div class="apireq">
<a href="/nodeinfo">http://[31a:fb8a:c43e:ca59::2]/nodeinfo</a>
</div>
<div class="wide"></div>
<small>Made with <a href="https://github.com/yakamok/Niflheim-api">Niflheim-API</a> by yakamok</small>
</div>
</body>
</html> </html>

6
api/yggapi.service
Unescape View File

@ -3,9 +3,11 @@ Description=yggapi
After=network.target After=network.target
[Service] [Service]
User=yakamo WorkingDirectory=/opt/yggdrasil-api/api
ExecStart=/usr/bin/python /opt/yggapi.py ExecStart=/opt/yggdrasil-api/venv/bin/python /opt/yggdrasil-api/api/niflheim-api.py
Restart=always Restart=always
EnvironmentFile=/opt/yggdrasil-api/venv/bin/activate
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

4
requirements.txt
Unescape View File

@ -0,0 +1,4 @@
flask
flask_restful
psycopg2
requests
Write Preview
Loading…
Cancel
Save