r4sas/yggdrasil-api
1
0
Fork 0
mirror of https://github.com/r4sas/Niflheim-api synced 2025-09-03 17:41:52 +00:00
Code Issues Releases Wiki Activity

correction updates, make it usable

Browse Source
This commit is contained in:
R4SAS 2019-12-31 22:50:44 +00:00
parent d9a062362b
commit ce51013715
8 changed files with 167 additions and 138 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

59
api/crawler.py
View File

@ -1,6 +1,8 @@
#some of this code was contributed by Arcelier #!/usr/bin/env python
#original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
#multithreaded by neilalexander # some of this code was contributed by Arcelier
# original code https://github.com/Arceliar/yggdrasil-map/blob/master/scripts/crawl-dht.py
# multithreaded by neilalexander
import psycopg2 import psycopg2
import json import json
@ -12,6 +14,20 @@ import traceback
from threading import Lock, Thread from threading import Lock, Thread
from Queue import Queue from Queue import Queue
# Configuration to use TCP connection or unix domain socket for admin connection to yggdrasil
useAdminSock = True
yggAdminTCP = ('localhost', 9001)
yggAdminSock = ('/var/run/yggdrasil.sock')
DB_PASSWORD = "password"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
## Save in database node info fields like buildname, buildarch, buildplatform, buildversion (True/False)?
saveDefaultNodeInfo = False
removableFileds = ['buildname', 'buildarch', 'buildplatform', 'buildversion']
class Worker(Thread): class Worker(Thread):
def __init__(self, tasks): def __init__(self, tasks):
Thread.__init__(self) Thread.__init__(self)
@ -48,13 +64,6 @@ nodeinfo = dict()
nodeinfomutex = Lock() nodeinfomutex = Lock()
nodeinfopool = ThreadPool(30) nodeinfopool = ThreadPool(30)
host_port = ('localhost', 9001)
DB_PASSWORD = "akemi2501"
DB_USER = "yggindex"
DB_NAME = "yggindex"
DB_HOST = "localhost"
def recv_until_done(soc): def recv_until_done(soc):
all_data = [] all_data = []
while True: while True:
@ -83,8 +92,13 @@ def getNodeInfoTask(address, info):
def doRequest(req): def doRequest(req):
try: try:
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM) if useAdminSock:
ygg.connect(host_port) ygg = socket.socket(socket.AF_UNIX, socket.SOCK_STREAM)
ygg.connect(yggAdminSock)
else:
ygg = socket.socket(socket.AF_INET, socket.SOCK_STREAM)
ygg.connect(yggAdminTCP)
ygg.send(req) ygg.send(req)
data = json.loads(recv_until_done(ygg)) data = json.loads(recv_until_done(ygg))
return data return data
@ -118,24 +132,34 @@ def insert_new_entry(ipv6, coords):
nodejson = "{}" nodejson = "{}"
if ipv6 in nodeinfo: if ipv6 in nodeinfo:
with nodeinfomutex: with nodeinfomutex:
if not saveDefaultNodeInfo:
# remove default Node info fields
for field in removableFileds:
tmprm = nodeinfo[ipv6].pop(field, None)
nodejson = json.dumps(nodeinfo[ipv6]) nodejson = json.dumps(nodeinfo[ipv6])
nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else "" nodename = nodeinfo[ipv6]["name"] if "name" in nodeinfo[ipv6] else ""
dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD) dbconn = psycopg2.connect(host=DB_HOST, database=DB_NAME, user=DB_USER, password=DB_PASSWORD)
cur = dbconn.cursor() cur = dbconn.cursor()
timestamp = str(int(time.time()))
cur.execute( cur.execute(
"INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;", "INSERT INTO yggindex (ipv6, coords, unixtstamp, name) VALUES(%s, %s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET unixtstamp=%s, coords=%s, name=%s;",
(ipv6, coords, str(int(time.time())), nodename, str(int(time.time())), coords, nodename) (ipv6, coords, timestamp, nodename, timestamp, coords, nodename)
) )
cur.execute( cur.execute(
"INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, NOW()) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=NOW();", "INSERT INTO yggnodeinfo (ipv6, nodeinfo, timestamp) VALUES(%s, %s, %s) ON CONFLICT (ipv6) DO UPDATE SET nodeinfo=%s, timestamp=%s;",
(ipv6, nodejson, nodejson) (ipv6, nodejson, timestamp, nodejson, timestamp)
) )
dbconn.commit() dbconn.commit()
cur.close() cur.close()
dbconn.close() dbconn.close()
except Exception as e: except Exception as e:
print("database error inserting") print "database error inserting"
traceback.print_exc() traceback.print_exc()
def handleNodeInfo(address, data): def handleNodeInfo(address, data):
global nodeinfo global nodeinfo
@ -183,6 +207,7 @@ for k,v in selfInfo['response']['self'].iteritems():
# Loop over rumored nodes and ping them, adding to visited if they respond # Loop over rumored nodes and ping them, adding to visited if they respond
while len(rumored) > 0: while len(rumored) > 0:
for k,v in rumored.iteritems(): for k,v in rumored.iteritems():
#print "Processing", v['coords']
handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords']))) handleResponse(k, v, doRequest(getDHTPingRequest(v['box_pub_key'], v['coords'])))
break break
del rumored[k] del rumored[k]

3
api/max-min.py
View File

@ -1,4 +1,7 @@
#!/usr/bin/env python
#max/min for the day with nodes #max/min for the day with nodes
import psycopg2 import psycopg2
import time import time

94
api/niflheim-api.py
View File

@ -1,8 +1,11 @@
#!/usr/bin/env python
import time import time
from flask import Flask, render_template from flask import Flask, render_template
from flask_restful import Resource, Api from flask_restful import Resource, Api
import requests import requests
import psycopg2 import psycopg2
import json
app = Flask(__name__) app = Flask(__name__)
api = Api(app) api = Api(app)
@ -12,36 +15,16 @@ DB_USER = "yggindex"
DB_NAME = "yggindex" DB_NAME = "yggindex"
DB_HOST = "localhost" DB_HOST = "localhost"
#will add this as something seperate and pull down to a file rather ALIVE_SECONDS = 3600 # 1 hour
#than request it everytime.
def get_nodelist():
data = requests.get("use the raw view of the github nodelist", timeout=1)
nodes = [x.split() for x in data.text.split('\n') if x]
index_table = {}
for key in nodes:
index_table[key[0]] = key[1]
return index_table
def check_nodelist(nodetable, key):
if nodetable:
if nodetable.get(key):
return nodetable.get(key)
else:
return key
else:
return key
def age_calc(ustamp): def age_calc(ustamp):
if (time.time() - ustamp) <= 14400: if (time.time() - ustamp) <= ALIVE_SECONDS:
return True return True
else: else:
return False return False
#active nodes in the past 4hrs # active nodes
class nodesCurrent(Resource): class nodesCurrent(Resource):
def get(self): def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\ dbconn = psycopg2.connect(host=DB_HOST,\
@ -65,6 +48,30 @@ class nodesCurrent(Resource):
return nodelist return nodelist
# nodes info
class nodesInfo(Resource):
def get(self):
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
nodes = {}
cur.execute("select * from yggnodeinfo")
for i in cur.fetchall():
if age_calc(int(i[2])):
nodes[i[0]] = json.loads(i[1])
dbconn.commit()
cur.close()
dbconn.close()
nodeinfo = {}
nodeinfo['yggnodeinfo'] = nodes
return nodeinfo
@app.route("/") @app.route("/")
def fpage(): def fpage():
dbconn = psycopg2.connect(host=DB_HOST,\ dbconn = psycopg2.connect(host=DB_HOST,\
@ -72,56 +79,23 @@ def fpage():
user=DB_USER,\ user=DB_USER,\
password=DB_PASSWORD) password=DB_PASSWORD)
cur = dbconn.cursor() cur = dbconn.cursor()
nodes = {} nodes = 0
cur.execute("select * from yggindex") cur.execute("select * from yggindex")
for i in cur.fetchall(): for i in cur.fetchall():
if age_calc(int(i[2])): if age_calc(int(i[2])):
nodes[i[0]] = [i[1], int(i[2])] nodes += 1
dbconn.commit() dbconn.commit()
cur.close() cur.close()
dbconn.close() dbconn.close()
return render_template('index.html', nodes=str(len(nodes))) return render_template('index.html', nodes=nodes)
@app.route("/contrib")
def cpage():
try:
domain_nodelist = get_nodelist()
print "list exists"
except:
print "failed"
domain_nodelist = None
dbconn = psycopg2.connect(host=DB_HOST,\
database=DB_NAME,\
user=DB_USER,\
password=DB_PASSWORD)
cur = dbconn.cursor()
cur.execute("select * from contrib")
nodes = []
for i in cur.fetchall():
if age_calc(int(i[1])):
nodes.append(i[0])
dbconn.commit()
cur.close()
dbconn.close()
dnodes = []
for key in nodes:
dnodes.append(check_nodelist(domain_nodelist, key))
dnodes.sort(reverse=True)
return render_template('contrib.html', contribnodes=dnodes, nocontribs=str(len(dnodes)))
#sort out the api request here for the url #sort out the api request here for the url
api.add_resource(nodesCurrent, '/current') api.add_resource(nodesCurrent, '/current')
api.add_resource(nodesInfo, '/nodeinfo')
if __name__ == '__main__': if __name__ == '__main__':
app.run(host='::', port=3000) app.run(host='::', port=3000)

78
api/static/style.css
View File

@ -1,28 +1,56 @@
/* style sheet for api pages */ /* style sheet for api pages */
body, input, textarea { body, input, textarea {
font-family: sans-serif; font-family: sans-serif;
padding: 0; padding: 0;
margin: 0; margin: 0;
background-color: #F0F0F0; background-color: #F0F0F0;
}a { }
text-decoration:none;
color:#800000; a {
}h2 { text-decoration: none;
font-size:18px; color: #800000;
}#title { }
max-width:570px;
margin-bottom:40px; h2 {
line-height: 35px; font-size: 18px;
font-size: 25px; }
font-weight: bold;
padding-top:20px; #title {
padding-bottom:5px; max-width: 570px;
border-bottom:1px solid #c4c4c4; margin-bottom: 40px;
}#wrapper { line-height: 35px;
font-size: 15px; font-size: 25px;
max-width: 570px; font-weight: bold;
margin: auto; padding-top: 20px;
}.contribs { padding-bottom: 5px;
font-size:13px; border-bottom: 1px solid #c4c4c4;
margin-bottom:6px; }
#title a {
color:#302f30;
}
#wrapper {
font-size: 15px;
max-width: 570px;
margin: auto;
}
.apireq {
padding-top: 5px;
padding-left: 10px;
padding-bottom: 5px;
line-height: 25px;
background-color: #e5e5e5;
color: #4c4c4c;
border-top: solid 1px #d3d3d3;
border-bottom: solid 1px #d3d3d3;
margin-top: 8px;
margin-bottom: 20px;
}
.wide {
width: 100%;
border-bottom: solid 1px #d3d3d3;
margin-bottom: 25px;
} }

21
api/templates/contrib.html
View File

@ -1,21 +0,0 @@
<!doctype html>
<title>API</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
<html>
<body>
<div id="wrapper">
<div id="title">
<a href='/'>Yggdrasil Network API</a>
</div>
<h2>Contributors - {{nocontribs}}</h2>
<p>This is a list of contributors sending in their view of the network, this list is based on the contributions sent in the last 4 hours.</p><br />
{% for x in contribnodes %}
<div class="contribs">{{ x }}</div>
{% endfor %}
</div>
</body>
</html>

40
api/templates/index.html
View File

@ -2,22 +2,36 @@
<title>API</title> <title>API</title>
<link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/> <link rel="stylesheet" type="text/css" href="{{ url_for('static', filename='style.css')}}"/>
<html> <html>
<body> <body>
<div id="wrapper"> <div id="wrapper">
<div id="title"> <div id="title">
<a href='/'>Yggdrasil Network API</a> <a href='/'>Yggdrasil Network API</a>
</div> </div>
This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br /><br /> This API provides collected DHT views from different nodes around the network in an effort to let yggdrasil developers and others research the networks growth.<br /><br />
<strong>Current Nodes online:</strong> {{ nodes }}<br /><br /> <div class="wide"></div>
<center>
<strong>Current Nodes online<br />
<font size="18">{{ nodes }}</font></strong>
</center>
<br /><br />
<a href="http://[21f:dd73:7cdb:773b:a924:7ec0:800b:221e]/">Yggdrasil Interactive World map</a><br /><br />
<a href="static/map.svg">Yggdrasil World map</a><br/><br/> <div class="wide"></div>
<a href="/contrib">Contributors</a><br /><br /><br /> <strong>Make an API request</strong><br />
<strong>Make an API request:</strong><br/><br/> <small>note: data updated every 15 minutes, all requests are returned in JSON.</small><br /><br />
<a href="/current">http://y.yakamo.org:3000/current</a><br/><br/><br/>
To contribute to the yggdrasil world map, you can send your view of the network using <a href="https://github.com/yakamok/ygg-node-db/blob/master/send-view.py">send-view.py</a>. Set a Cron job to once an hour.<br/><br/>
Thanks to Arceliar for the map generating script. Get a current list of active and online nodes:<br />
</div> <div class="apireq">
</body> <a href="/current">http://[31a:fb8a:c43e:ca59::2]/current</a>
</div>
Nodeino from all current active nodes:<br />
<div class="apireq">
<a href="/nodeinfo">http://[31a:fb8a:c43e:ca59::2]/nodeinfo</a>
</div>
<div class="wide"></div>
<small>Made with <a href="https://github.com/yakamok/Niflheim-api">Niflheim-API</a> by yakamok</small>
</div>
</body>
</html> </html>

6
api/yggapi.service
View File

@ -3,9 +3,11 @@ Description=yggapi
After=network.target After=network.target
[Service] [Service]
User=yakamo WorkingDirectory=/opt/yggdrasil-api/api
ExecStart=/usr/bin/python /opt/yggapi.py ExecStart=/opt/yggdrasil-api/venv/bin/python /opt/yggdrasil-api/api/niflheim-api.py
Restart=always Restart=always
EnvironmentFile=/opt/yggdrasil-api/venv/bin/activate
[Install] [Install]
WantedBy=multi-user.target WantedBy=multi-user.target

4
requirements.txt Normal file
View File

@ -0,0 +1,4 @@
flask
flask_restful
psycopg2
requests