mirror of
https://github.com/r4sas/Niflheim-api
synced 2025-01-25 22:24:13 +00:00
correction updates, make it usable
This commit is contained in:
parent
d9a062362b
commit
ce51013715
@ -1,6 +1,8 @@
|
||||
#some of this code was contributed by Arcelier
|
||||
#original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
|
||||
#multithreaded by neilalexander
|
||||
#!/usr/bin/env python
|
||||
|
||||
# some of this code was contributed by Arcelier
|
||||
# original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
|
||||
# multithreaded by neilalexander
|
||||
|
||||
import psycopg2
|
||||
import json
|
||||
@ -12,6 +14,20 @@ import traceback
|
||||
from threading import Lock, Thread
|
||||
from Queue import Queue
|
||||
|
||||
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
|
||||
useAdminSock = True
|
||||
yggAdminTCP = ('localhost', 9001)
|
||||
yggAdminSock = ('/var/run/yggdrasil.sock')
|
||||
|
||||
DB_PASSWORD = "password"
|
||||
DB_USER = "yggindex"
|
||||
DB_NAME = "yggindex"
|
||||
DB_HOST = "localhost"
|
||||
|
||||
## Save in database node info fields like buildname, buildarch, buildplatform, buildversion (True/False)?
|
||||
saveDefaultNodeInfo = False
|
||||
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion']
|
||||
|
||||
class Worker(Thread):
|
||||
def __init__(self, tasks):
|
||||
Thread.__init__(self)
|
||||
@ -48,13 +64,6 @@ nodeinfo = dict()
|
||||
nodeinfomutex = Lock()
|
||||
nodeinfopool = ThreadPool(30)
|
||||
|
||||
host_port = ('localhost', 9001)
|
||||
|
||||
DB_PASSWORD = "akemi2501"
|
||||
DB_USER = "yggindex"
|
||||
DB_NAME = "yggindex"
|
||||
DB_HOST = "localhost"
|
||||
|
||||
def recv_until_done(soc):
|
||||
all_data = []
|
||||
while True:
|
||||
@ -83,8 +92,13 @@ def getNodeInfoTask(address, info):
|
||||
|
||||
def doRequest(req):
|
||||
try:
|
||||
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
ygg.connect(host_port)
|
||||
if useAdminSock:
|
||||
ygg = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
|
||||
ygg.connect(yggAdminSock)
|
||||
else:
|
||||
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
|
||||
ygg.connect(yggAdminTCP)
|
||||
|
||||
ygg.send(req)
|
||||
data = json.loads(recv_until_done(ygg))
|
||||
return data
|
||||
@ -118,24 +132,34 @@ def insert_new_entry(ipv6, coords):
|
||||
nodejson = "{}"
|
||||
if ipv6 in nodeinfo:
|
||||
with nodeinfomutex:
|
||||
|
||||
if not saveDefaultNodeInfo:
|
||||
# remove default Node info fields
|
||||
for field in removableFileds:
|
||||
tmprm = nodeinfo[ipv6].pop(field, None)
|
||||
|
||||
nodejson = json.dumps(nodeinfo[ipv6])
|
||||
nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
|
||||
|
||||
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
|
||||
cur = dbconn.cursor()
|
||||
timestamp = str(int(time.time()))
|
||||
|
||||
cur.execute(
|
||||
"INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
|
||||
(ipv6, coords, str(int(time.time())), nodename, str(int(time.time())), coords, nodename)
|
||||
(ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
|
||||
)
|
||||
cur.execute(
|
||||
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, NOW()) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=NOW();",
|
||||
(ipv6, nodejson, nodejson)
|
||||
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
|
||||
(ipv6, nodejson, timestamp, nodejson, timestamp)
|
||||
)
|
||||
|
||||
dbconn.commit()
|
||||
cur.close()
|
||||
dbconn.close()
|
||||
except Exception as e:
|
||||
print("database error inserting")
|
||||
traceback.print_exc()
|
||||
print "database error inserting"
|
||||
traceback.print_exc()
|
||||
|
||||
def handleNodeInfo(address, data):
|
||||
global nodeinfo
|
||||
@ -183,6 +207,7 @@ for k,v in selfInfo['response']['self'].iteritems():
|
||||
# Loop over rumored nodes and ping them, adding to visited if they respond
|
||||
while len(rumored) > 0:
|
||||
for k,v in rumored.iteritems():
|
||||
#print "Processing", v['coords']
|
||||
handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'])))
|
||||
break
|
||||
del rumored[k]
|
||||
|
||||
@ -1,4 +1,7 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
#max/min for the day with nodes
|
||||
|
||||
import psycopg2
|
||||
import time
|
||||
|
||||
|
||||
@ -1,8 +1,11 @@
|
||||
#!/usr/bin/env python
|
||||
|
||||
import time
|
||||
from flask import Flask, render_template
|
||||
from flask_restful import Resource, Api
|
||||
import requests
|
||||
import psycopg2
|
||||
import json
|
||||
|
||||
app = Flask(__name__)
|
||||
api = Api(app)
|
||||
@ -12,36 +15,16 @@ DB_USER = "yggindex"
|
||||
DB_NAME = "yggindex"
|
||||
DB_HOST = "localhost"
|
||||
|
||||
#will add this as something seperate and pull down to a file rather
|
||||
#than request it everytime.
|
||||
def get_nodelist():
|
||||
data = requests.get("use the raw view of the github nodelist", timeout=1)
|
||||
nodes = [x.split() for x in data.text.split('\n') if x]
|
||||
|
||||
index_table = {}
|
||||
|
||||
for key in nodes:
|
||||
index_table[key[0]] = key[1]
|
||||
return index_table
|
||||
|
||||
|
||||
def check_nodelist(nodetable, key):
|
||||
if nodetable:
|
||||
if nodetable.get(key):
|
||||
return nodetable.get(key)
|
||||
else:
|
||||
return key
|
||||
else:
|
||||
return key
|
||||
ALIVE_SECONDS = 3600 # 1 hour
|
||||
|
||||
|
||||
def age_calc(ustamp):
|
||||
if (time.time() - ustamp) <= 14400:
|
||||
if (time.time() - ustamp) <= ALIVE_SECONDS:
|
||||
return True
|
||||
else:
|
||||
return False
|
||||
|
||||
#active nodes in the past 4hrs
|
||||
# active nodes
|
||||
class nodesCurrent(Resource):
|
||||
def get(self):
|
||||
dbconn = psycopg2.connect(host=DB_HOST,\
|
||||
@ -65,6 +48,30 @@ class nodesCurrent(Resource):
|
||||
return nodelist
|
||||
|
||||
|
||||
# nodes info
|
||||
class nodesInfo(Resource):
|
||||
def get(self):
|
||||
dbconn = psycopg2.connect(host=DB_HOST,\
|
||||
database=DB_NAME,\
|
||||
user=DB_USER,\
|
||||
password=DB_PASSWORD)
|
||||
cur = dbconn.cursor()
|
||||
nodes = {}
|
||||
cur.execute("select * from yggnodeinfo")
|
||||
for i in cur.fetchall():
|
||||
if age_calc(int(i[2])):
|
||||
nodes[i[0]] = json.loads(i[1])
|
||||
|
||||
dbconn.commit()
|
||||
cur.close()
|
||||
dbconn.close()
|
||||
|
||||
nodeinfo = {}
|
||||
nodeinfo['yggnodeinfo'] = nodes
|
||||
|
||||
return nodeinfo
|
||||
|
||||
|
||||
@app.route("/")
|
||||
def fpage():
|
||||
dbconn = psycopg2.connect(host=DB_HOST,\
|
||||
@ -72,56 +79,23 @@ def fpage():
|
||||
user=DB_USER,\
|
||||
password=DB_PASSWORD)
|
||||
cur = dbconn.cursor()
|
||||
nodes = {}
|
||||
nodes = 0
|
||||
cur.execute("select * from yggindex")
|
||||
|
||||
for i in cur.fetchall():
|
||||
if age_calc(int(i[2])):
|
||||
nodes[i[0]] = [i[1], int(i[2])]
|
||||
nodes += 1
|
||||
|
||||
dbconn.commit()
|
||||
cur.close()
|
||||
dbconn.close()
|
||||
|
||||
return render_template('index.html', nodes=str(len(nodes)))
|
||||
|
||||
|
||||
@app.route("/contrib")
|
||||
def cpage():
|
||||
try:
|
||||
domain_nodelist = get_nodelist()
|
||||
print "list exists"
|
||||
except:
|
||||
print "failed"
|
||||
domain_nodelist = None
|
||||
|
||||
dbconn = psycopg2.connect(host=DB_HOST,\
|
||||
database=DB_NAME,\
|
||||
user=DB_USER,\
|
||||
password=DB_PASSWORD)
|
||||
cur = dbconn.cursor()
|
||||
cur.execute("select * from contrib")
|
||||
nodes = []
|
||||
|
||||
for i in cur.fetchall():
|
||||
if age_calc(int(i[1])):
|
||||
nodes.append(i[0])
|
||||
|
||||
dbconn.commit()
|
||||
cur.close()
|
||||
dbconn.close()
|
||||
|
||||
dnodes = []
|
||||
for key in nodes:
|
||||
dnodes.append(check_nodelist(domain_nodelist, key))
|
||||
|
||||
dnodes.sort(reverse=True)
|
||||
|
||||
return render_template('contrib.html', contribnodes=dnodes, nocontribs=str(len(dnodes)))
|
||||
return render_template('index.html', nodes=nodes)
|
||||
|
||||
|
||||
#sort out the api request here for the url
|
||||
api.add_resource(nodesCurrent, '/current')
|
||||
api.add_resource(nodesInfo, '/nodeinfo')
|
||||
|
||||
if __name__ == '__main__':
|
||||
app.run(host='::', port=3000)
|
||||
|
||||
@ -1,28 +1,56 @@
|
||||
/* style sheet for api pages */
|
||||
body, input, textarea {
|
||||
font-family: sans-serif;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
background-color: #F0F0F0;
|
||||
}a {
|
||||
text-decoration:none;
|
||||
color:#800000;
|
||||
}h2 {
|
||||
font-size:18px;
|
||||
}#title {
|
||||
max-width:570px;
|
||||
margin-bottom:40px;
|
||||
line-height: 35px;
|
||||
font-size: 25px;
|
||||
font-weight: bold;
|
||||
padding-top:20px;
|
||||
padding-bottom:5px;
|
||||
border-bottom:1px solid #c4c4c4;
|
||||
}#wrapper {
|
||||
font-size: 15px;
|
||||
max-width: 570px;
|
||||
margin: auto;
|
||||
}.contribs {
|
||||
font-size:13px;
|
||||
margin-bottom:6px;
|
||||
font-family: sans-serif;
|
||||
padding: 0;
|
||||
margin: 0;
|
||||
background-color: #F0F0F0;
|
||||
}
|
||||
|
||||
a {
|
||||
text-decoration: none;
|
||||
color: #800000;
|
||||
}
|
||||
|
||||
h2 {
|
||||
font-size: 18px;
|
||||
}
|
||||
|
||||
#title {
|
||||
max-width: 570px;
|
||||
margin-bottom: 40px;
|
||||
line-height: 35px;
|
||||
font-size: 25px;
|
||||
font-weight: bold;
|
||||
padding-top: 20px;
|
||||
padding-bottom: 5px;
|
||||
border-bottom: 1px solid #c4c4c4;
|
||||
}
|
||||
|
||||
#title a {
|
||||
color:#302f30;
|
||||
}
|
||||
|
||||
#wrapper {
|
||||
font-size: 15px;
|
||||
max-width: 570px;
|
||||
margin: auto;
|
||||
}
|
||||
|
||||
.apireq {
|
||||
padding-top: 5px;
|
||||
padding-left: 10px;
|
||||
padding-bottom: 5px;
|
||||
line-height: 25px;
|
||||
background-color: #e5e5e5;
|
||||
color: #4c4c4c;
|
||||
border-top: solid 1px #d3d3d3;
|
||||
border-bottom: solid 1px #d3d3d3;
|
||||
margin-top: 8px;
|
||||
margin-bottom: 20px;
|
||||
}
|
||||
|
||||
.wide {
|
||||
width: 100%;
|
||||
border-bottom: solid 1px #d3d3d3;
|
||||
margin-bottom: 25px;
|
||||
}
|
||||
|
||||
@ -1,21 +0,0 @@
|
||||
<!doctype html>
|
||||
<title>API</title>
|
||||
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wrapper">
|
||||
<div id="title">
|
||||
<a href='/'>Yggdrasil Network API</a>
|
||||
</div>
|
||||
|
||||
<h2>Contributors - {{nocontribs}}</h2>
|
||||
<p>This is a list of contributors sending in their view of the network, this list is based on the contributions sent in the last 4 hours.</p><br />
|
||||
|
||||
{% for x in contribnodes %}
|
||||
<div class="contribs">{{ x }}</div>
|
||||
{% endfor %}
|
||||
|
||||
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
@ -2,22 +2,36 @@
|
||||
<title>API</title>
|
||||
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
|
||||
<html>
|
||||
<body>
|
||||
<div id="wrapper">
|
||||
<body>
|
||||
<div id="wrapper">
|
||||
<div id="title">
|
||||
<a href='/'>Yggdrasil Network API</a>
|
||||
<a href='/'>Yggdrasil Network API</a>
|
||||
</div>
|
||||
This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br /><br />
|
||||
This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br />
|
||||
|
||||
<strong>Current Nodes online:</strong> {{ nodes }}<br /><br />
|
||||
<div class="wide"></div>
|
||||
<center>
|
||||
<strong>Current Nodes online<br />
|
||||
<font size="18">{{ nodes }}</font></strong>
|
||||
</center>
|
||||
<br /><br />
|
||||
<a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
|
||||
|
||||
<a href="static/map.svg">Yggdrasil World map</a><br/><br/>
|
||||
<a href="/contrib">Contributors</a><br /><br /><br />
|
||||
<strong>Make an API request:</strong><br/><br/>
|
||||
<a href="/current">http://y.yakamo.org:3000/current</a><br/><br/><br/>
|
||||
To contribute to the yggdrasil world map, you can send your view of the network using <a href="https://github.com/yakamok/ygg-node-db/blob/master/send-view.py">send-view.py</a>. Set a Cron job to once an hour.<br/><br/>
|
||||
<div class="wide"></div>
|
||||
<strong>Make an API request</strong><br />
|
||||
<small>note: data updated every 15 minutes, all requests are returned in JSON.</small><br /><br />
|
||||
|
||||
Thanks to Arceliar for the map generating script.
|
||||
</div>
|
||||
</body>
|
||||
Get a current list of active and online nodes:<br />
|
||||
<div class="apireq">
|
||||
<a href="/current">http://[31a:fb8a:c43e:ca59::2]/current</a>
|
||||
</div>
|
||||
Nodeino from all current active nodes:<br />
|
||||
<div class="apireq">
|
||||
<a href="/nodeinfo">http://[31a:fb8a:c43e:ca59::2]/nodeinfo</a>
|
||||
</div>
|
||||
|
||||
<div class="wide"></div>
|
||||
<small>Made with <a href="https://github.com/yakamok/Niflheim-api">Niflheim-API</a> by yakamok</small>
|
||||
</div>
|
||||
</body>
|
||||
</html>
|
||||
|
||||
@ -3,9 +3,11 @@ Description=yggapi
|
||||
After=network.target
|
||||
|
||||
[Service]
|
||||
User=yakamo
|
||||
ExecStart=/usr/bin/python /opt/yggapi.py
|
||||
WorkingDirectory=/opt/yggdrasil-api/api
|
||||
ExecStart=/opt/yggdrasil-api/venv/bin/python /opt/yggdrasil-api/api/niflheim-api.py
|
||||
Restart=always
|
||||
|
||||
EnvironmentFile=/opt/yggdrasil-api/venv/bin/activate
|
||||
|
||||
[Install]
|
||||
WantedBy=multi-user.target
|
||||
|
||||
4
requirements.txt
Normal file
4
requirements.txt
Normal file
@ -0,0 +1,4 @@
|
||||
flask
|
||||
flask_restful
|
||||
psycopg2
|
||||
requests
|
||||
Loading…
x
Reference in New Issue
Block a user