Browse Source

Major refactor of the codebase to support multiple networks

master
Lyndsay Roger 9 years ago
parent
commit
9b5240c03b
  1. 25
      README.md
  2. 12
      configs/bitcoin-test.json
  3. 12
      configs/bitcoin.json
  4. 12
      configs/dnsseeder.json
  5. 12
      configs/twister.json
  6. 134
      crawler.go
  7. 143
      dns.go
  8. 329
      http.go
  9. 181
      main.go
  10. 181
      network.go
  11. 12
      node.go
  12. 139
      seeder.go

25
README.md

@ -1,9 +1,11 @@ @@ -1,9 +1,11 @@
# dnsseeder
Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/)
Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/) and the Bitcoin networks.
It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder
It is based on the original c++ seeders created for the Bitcoin network and copied to other similar networks.
This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option.
This application can seed one or more networks on the same ip address. At the moment there are config files for the Twister and Bitcoin networks. You use the -netfile commandline option to specify one or more comma seperated filenames to load the network configuration for that network.
You can use the -j option to produce a sample json network config file (dnsseeder.json) in the current directory and then edit the file to seed your own network.
Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/)
@ -34,19 +36,19 @@ The binary will then be available in ${HOME}/go/bin @@ -34,19 +36,19 @@ The binary will then be available in ${HOME}/go/bin
## Usage
$ dnsseeder -h <domain respond to> -net <network to seed>
$ dnsseeder -v -netfile <filename1,filename2>
An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running.
If you want to be able to view the web interface then add -w port for the web server to listen on. If this is not provided then no web interface will be available. With the web site running you can then access the site by http://localhost:port/statusCG
If you want to be able to view the web interface then add -w port for the web server to listen on. If this is not provided then no web interface will be available. With the web site running you can then access the site by http://localhost:port/summary
**NOTE -** For security reasons the web server will only listen on localhost so you will need to either use an ssh tunnel or proxy requests via a web server like Nginx or Apache.
```
Command line Options:
-h hostname to serve
-net The network to seed. Currently twister, bitcoin, bitcoin-test
-netfile comma seperated list of json network config files to load
-j write a sample network config file in json format and exit.
-p port to listen on for DNS requests
-d Produce debug output
-v Produce verbose output
@ -66,18 +68,11 @@ mkdir -p ${LOGDIR} @@ -66,18 +68,11 @@ mkdir -p ${LOGDIR}
gzip ${LOGDIR}/*.log
# pass through the logging level needed
if [ -z ${1} ]; then
THENET="twister"
else
THENET="${1}"
fi
cd
echo
echo "======= Run the Go Language dnsseed ======="
echo
${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <dns.port.to.listen.on> ${THENET} -v -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
${HOME}/go/bin/dnsseeder -p <dns.port.to.listen.on> -v -w 8880 -netfile ${1} 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
```

12
configs/bitcoin-test.json

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
{
"Name": "BitcoinNet-Test",
"Desc": "Bitcoin Test Net",
"Id": "0xdab5bffa",
"Port": 18333,
"Pver": 70001,
"DNSName": "btctseed.zagbot.com",
"TTL": 300,
"Seeder1": "testnet-seed.alexykot.me",
"Seeder2": "testnet-seed.bitcoin.petertodd.org",
"Seeder3": "testnet-seed.bluematt.me"
}

12
configs/bitcoin.json

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
{
"Name": "BitcoinNet",
"Desc": "Bitcoin Main Net",
"Id": "0xd9b4bef9",
"Port": 8333,
"Pver": 70001,
"DNSName": "btcseed.zagbot.com",
"TTL": 600,
"Seeder1": "dnsseed.bluematt.me",
"Seeder2": "bitseed.xf2.org",
"Seeder3": "dnsseed.bitcoin.dashjr.org"
}

12
configs/dnsseeder.json

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
{
"Name": "SeederNet",
"Desc": "Description of SeederNet",
"Id": "0xabcdef01",
"Port": 1234,
"Pver": 70001,
"DNSName": "seeder.example.com",
"TTL": 600,
"Seeder1": "seeder1.example.com",
"Seeder2": "seed1.bob.com",
"Seeder3": "seed2.example.com"
}

12
configs/twister.json

@ -0,0 +1,12 @@ @@ -0,0 +1,12 @@
{
"Name": "TwisterNet",
"Desc": "Peer to Peer Microblogging Network",
"Id": "0xd2bbdaf0",
"Port": 28333,
"Pver": 60000,
"DNSName": "dnsseed.zagbot.com",
"TTL": 600,
"Seeder1": "seed2.twister.net.co",
"Seeder2": "seed.twister.net.co",
"Seeder3": "seed3.twister.net.co"
}

134
crawler.go

@ -20,82 +20,83 @@ func (e *crawlError) Error() string { @@ -20,82 +20,83 @@ func (e *crawlError) Error() string {
return "err: " + e.errLoc + ": " + e.Err.Error()
}
// crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
// crawlNode runs in a goroutine, crawls the remote ip and updates the master
// list of currently active addresses
func crawlTwistee(s *dnsseeder, tw *twistee) {
func crawlNode(s *dnsseeder, nd *node) {
tw.crawlActive = true
tw.crawlStart = time.Now()
nd.crawlActive = true
nd.crawlStart = time.Now()
defer crawlEnd(tw)
defer crawlEnd(nd)
if config.debug {
log.Printf("debug - start crawl: twistee %s status: %v:%v lastcrawl: %s\n",
net.JoinHostPort(tw.na.IP.String(),
strconv.Itoa(int(tw.na.Port))),
tw.status,
tw.rating,
time.Since(tw.crawlStart).String())
log.Printf("debug - start crawl: node %s status: %v:%v lastcrawl: %s\n",
net.JoinHostPort(nd.na.IP.String(),
strconv.Itoa(int(nd.na.Port))),
nd.status,
nd.rating,
time.Since(nd.crawlStart).String())
}
// connect to the remote ip and ask them for their addr list
rna, e := crawlIP(s.net.pver, s.net.id, tw)
rna, e := crawlIP(s.pver, s.id, nd)
if e != nil {
// update the fact that we have not connected to this twistee
tw.lastTry = time.Now()
tw.connectFails++
tw.statusStr = e.Error()
// update the fact that we have not connected to this node
nd.lastTry = time.Now()
nd.connectFails++
nd.statusStr = e.Error()
// update the status of this failed twistee
switch tw.status {
// update the status of this failed node
switch nd.status {
case statusRG:
// if we are full then any RG failures will skip directly to NG
if s.isFull() {
tw.status = statusNG // not able to connect to this twistee so ignore
tw.statusTime = time.Now()
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
} else {
if tw.rating += 25; tw.rating > 30 {
tw.status = statusWG
tw.statusTime = time.Now()
if nd.rating += 25; nd.rating > 30 {
nd.status = statusWG
nd.statusTime = time.Now()
}
}
case statusCG:
if tw.rating += 25; tw.rating >= 50 {
tw.status = statusWG
tw.statusTime = time.Now()
if nd.rating += 25; nd.rating >= 50 {
nd.status = statusWG
nd.statusTime = time.Now()
}
case statusWG:
if tw.rating += 15; tw.rating >= 100 {
tw.status = statusNG // not able to connect to this twistee so ignore
tw.statusTime = time.Now()
if nd.rating += 15; nd.rating >= 100 {
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
}
}
// no more to do so return which will shutdown the goroutine & call
// the deffered cleanup
if config.verbose {
log.Printf("status - failed crawl: twistee %s s:r:f: %v:%v:%v %s\n",
net.JoinHostPort(tw.na.IP.String(),
strconv.Itoa(int(tw.na.Port))),
tw.status,
tw.rating,
tw.connectFails,
tw.statusStr)
log.Printf("%s: failed crawl node: %s s:r:f: %v:%v:%v %s\n",
s.name,
net.JoinHostPort(nd.na.IP.String(),
strconv.Itoa(int(nd.na.Port))),
nd.status,
nd.rating,
nd.connectFails,
nd.statusStr)
}
return
}
// succesful connection and addresses received so mark status
if tw.status != statusCG {
tw.status = statusCG
tw.statusTime = time.Now()
if nd.status != statusCG {
nd.status = statusCG
nd.statusTime = time.Now()
}
cs := tw.lastConnect
tw.rating = 0
tw.connectFails = 0
tw.lastConnect = time.Now()
tw.lastTry = time.Now()
tw.statusStr = "ok: received remote address list"
cs := nd.lastConnect
nd.rating = 0
nd.connectFails = 0
nd.lastConnect = time.Now()
nd.lastTry = time.Now()
nd.statusStr = "ok: received remote address list"
added := 0
@ -111,15 +112,16 @@ func crawlTwistee(s *dnsseeder, tw *twistee) { @@ -111,15 +112,16 @@ func crawlTwistee(s *dnsseeder, tw *twistee) {
}
if config.verbose {
log.Printf("status - crawl done: twistee: %s s:r:f: %v:%v:%v addr: %v:%v CrawlTime: %s Last connect: %v ago\n",
net.JoinHostPort(tw.na.IP.String(),
strconv.Itoa(int(tw.na.Port))),
tw.status,
tw.rating,
tw.connectFails,
log.Printf("%s: crawl done: node: %s s:r:f: %v:%v:%v addr: %v:%v CrawlTime: %s Last connect: %v ago\n",
s.name,
net.JoinHostPort(nd.na.IP.String(),
strconv.Itoa(int(nd.na.Port))),
nd.status,
nd.rating,
nd.connectFails,
len(rna),
added,
time.Since(tw.crawlStart).String(),
time.Since(nd.crawlStart).String(),
time.Since(cs).String())
}
@ -127,17 +129,17 @@ func crawlTwistee(s *dnsseeder, tw *twistee) { @@ -127,17 +129,17 @@ func crawlTwistee(s *dnsseeder, tw *twistee) {
}
// crawlEnd is a deffered func to update theList after a crawl is all done
func crawlEnd(tw *twistee) {
tw.crawlActive = false
// FIXME - scan for long term crawl active twistees. Dial timeout is 10 seconds
func crawlEnd(nd *node) {
nd.crawlActive = false
// FIXME - scan for long term crawl active node. Dial timeout is 10 seconds
// so should be done in under 5 minutes
}
// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) {
func crawlIP(pver uint32, netID wire.BitcoinNet, nd *node) ([]*wire.NetAddress, *crawlError) {
ip := tw.na.IP.String()
port := strconv.Itoa(int(tw.na.Port))
ip := nd.na.IP.String()
port := strconv.Itoa(int(nd.na.Port))
// get correct formatting for ipv6 addresses
dialString := net.JoinHostPort(ip, port)
@ -151,7 +153,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres @@ -151,7 +153,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres
defer conn.Close()
if config.debug {
log.Printf("debug - Connected to remote address: %s Last connect was %v ago\n", ip, time.Since(tw.lastConnect).String())
log.Printf("debug - Connected to remote address: %s Last connect was %v ago\n", ip, time.Since(nd.lastConnect).String())
}
// set a deadline for all comms to be done by. After this all i/o will error
@ -183,14 +185,14 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres @@ -183,14 +185,14 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres
if config.debug {
log.Printf("%s - Remote version: %v\n", ip, msg.ProtocolVersion)
}
// fill the Twistee struct with the remote details
tw.version = msg.ProtocolVersion
tw.services = msg.Services
tw.lastBlock = msg.LastBlock
if tw.strVersion != msg.UserAgent {
// fill the node struct with the remote details
nd.version = msg.ProtocolVersion
nd.services = msg.Services
nd.lastBlock = msg.LastBlock
if nd.strVersion != msg.UserAgent {
// if the srtVersion is already the same then don't overwrite it.
// saves the GC having to cleanup a perfectly good string
tw.strVersion = msg.UserAgent
nd.strVersion = msg.UserAgent
}
default:
return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")}
@ -231,7 +233,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres @@ -231,7 +233,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres
dowhile := true
for dowhile == true {
// Using the Bitcoin lib for the Twister Net means it does not understand some
// Using the Bitcoin lib for the some networks means it does not understand some
// of the commands and will error. We can ignore these as we are only
// interested in the addr message and its content.
msgaddr, _, _ := wire.ReadMessage(conn, pver, netID)

143
dns.go

@ -2,29 +2,11 @@ package main @@ -2,29 +2,11 @@ package main
import (
"log"
"sync"
// "sync"
"github.com/miekg/dns"
)
type currentIPs struct {
ipv4std []dns.RR
ipv4non []dns.RR
ipv6std []dns.RR
ipv6non []dns.RR
mtx sync.RWMutex
}
// latest holds the slices of current ip addresses
var latest currentIPs
// getLatestaRR returns a pointer to the latest slice of current dns.RR type
// dns.A records to pass back to the remote client
func getv4stdRR() []dns.RR { return latest.ipv4std }
func getv4nonRR() []dns.RR { return latest.ipv4non }
func getv6stdRR() []dns.RR { return latest.ipv6std }
func getv6nonRR() []dns.RR { return latest.ipv6non }
// updateDNS updates the current slices of dns.RR so incoming requests get a
// fast answer
func updateDNS(s *dnsseeder) {
@ -35,60 +17,61 @@ func updateDNS(s *dnsseeder) { @@ -35,60 +17,61 @@ func updateDNS(s *dnsseeder) {
// loop over each dns recprd type we need
for t := range []int{dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non} {
// FIXME above needs to be convertwd into one scan of theList if possible
numRR := 0
for _, tw := range s.theList {
for _, nd := range s.theList {
// when we reach max exit
if numRR >= 25 {
break
}
if tw.status != statusCG {
if nd.status != statusCG {
continue
}
if t == dnsV4Std || t == dnsV4Non {
if t == dnsV4Std && tw.dnsType == dnsV4Std {
if t == dnsV4Std && nd.dnsType == dnsV4Std {
r := new(dns.A)
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.na.IP
r.Hdr = dns.RR_Header{Name: s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl}
r.A = nd.na.IP
rr4std = append(rr4std, r)
numRR++
}
// if the twistee is using a non standard port then add the encoded port info to DNS
if t == dnsV4Non && tw.dnsType == dnsV4Non {
// if the node is using a non standard port then add the encoded port info to DNS
if t == dnsV4Non && nd.dnsType == dnsV4Non {
r := new(dns.A)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.na.IP
r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl}
r.A = nd.na.IP
rr4non = append(rr4non, r)
numRR++
r = new(dns.A)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.nonstdIP
r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl}
r.A = nd.nonstdIP
rr4non = append(rr4non, r)
numRR++
}
}
if t == dnsV6Std || t == dnsV6Non {
if t == dnsV6Std && tw.dnsType == dnsV6Std {
if t == dnsV6Std && nd.dnsType == dnsV6Std {
r := new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.na.IP
r.Hdr = dns.RR_Header{Name: s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl}
r.AAAA = nd.na.IP
rr6std = append(rr6std, r)
numRR++
}
// if the twistee is using a non standard port then add the encoded port info to DNS
if t == dnsV6Non && tw.dnsType == dnsV6Non {
// if the node is using a non standard port then add the encoded port info to DNS
if t == dnsV6Non && nd.dnsType == dnsV6Non {
r := new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.na.IP
r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl}
r.AAAA = nd.na.IP
rr6non = append(rr6non, r)
numRR++
r = new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.nonstdIP
r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl}
r.AAAA = nd.nonstdIP
rr6non = append(rr6non, r)
numRR++
}
@ -100,32 +83,32 @@ func updateDNS(s *dnsseeder) { @@ -100,32 +83,32 @@ func updateDNS(s *dnsseeder) {
s.mtx.RUnlock()
latest.mtx.Lock()
config.dnsmtx.Lock()
// update the map holding the details for this seeder
for t := range []int{dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non} {
switch t {
case dnsV4Std:
latest.ipv4std = rr4std
config.dns[s.dnsHost+".A"] = rr4std
case dnsV4Non:
latest.ipv4non = rr4non
config.dns["nonstd."+s.dnsHost+".A"] = rr4non
case dnsV6Std:
latest.ipv6std = rr6std
config.dns[s.dnsHost+".AAAA"] = rr6std
case dnsV6Non:
latest.ipv6non = rr6non
config.dns["nonstd."+s.dnsHost+".AAAA"] = rr6non
}
}
latest.mtx.Unlock()
config.dnsmtx.Unlock()
if config.debug {
log.Printf("debug - DNS update complete - rr4std: %v rr4non: %v rr6std: %v rr6non: %v\n", len(rr4std), len(rr4non), len(rr6std), len(rr6non))
}
}
// handleDNSStd processes a DNS request from remote client and returns
// handleDNS processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the standard port
func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
func handleDNS(w dns.ResponseWriter, r *dns.Msg) {
m := &dns.Msg{MsgHdr: dns.MsgHdr{
Authoritative: true,
@ -137,67 +120,31 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) { @@ -137,67 +120,31 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
switch r.Question[0].Qtype {
case dns.TypeA:
latest.mtx.RLock()
m.Answer = getv4stdRR()
latest.mtx.RUnlock()
qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Std)
case dns.TypeAAAA:
latest.mtx.RLock()
m.Answer = getv6stdRR()
latest.mtx.RUnlock()
qtype = "AAAA"
go updateDNSCounts(dnsV6Std)
case dns.TypeTXT:
qtype = "TXT"
case dns.TypeMX:
qtype = "MX"
case dns.TypeNS:
qtype = "NS"
default:
// return no answer to all other queries
}
w.WriteMsg(m)
if config.debug {
log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
qtype = "UNKNOWN"
}
}
// handleDNSNon processes a DNS request from remote client and returns
// a list of current ip addresses that the crawlers consider current.
// This function returns addresses that use the non standard port
func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
m := &dns.Msg{MsgHdr: dns.MsgHdr{
Authoritative: true,
RecursionAvailable: false,
}}
m.SetReply(r)
var qtype string
switch r.Question[0].Qtype {
case dns.TypeA:
latest.mtx.RLock()
m.Answer = getv4nonRR()
latest.mtx.RUnlock()
qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Non)
case dns.TypeAAAA:
latest.mtx.RLock()
m.Answer = getv6nonRR()
latest.mtx.RUnlock()
qtype = "AAAA"
go updateDNSCounts(dnsV6Non)
default:
// return no answer to all other queries
}
config.dnsmtx.RLock()
// if the dns map does not have a key for the request it will return an empty slice
m.Answer = config.dns[r.Question[0].Name+qtype]
config.dnsmtx.RUnlock()
w.WriteMsg(m)
if config.debug {
log.Printf("debug - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
}
// update the stats in a goroutine
go updateDNSCounts(r.Question[0].Name, qtype)
}
// serve starts the requested DNS server listening on the requested port

329
http.go

@ -13,12 +13,13 @@ import ( @@ -13,12 +13,13 @@ import (
// to the dnsseeder
func startHTTP(port string) {
http.HandleFunc("/dns", dnsHandler)
http.HandleFunc("/twistee", twisteeHandler)
http.HandleFunc("/dns", dnsWebHandler)
http.HandleFunc("/node", nodeHandler)
http.HandleFunc("/statusRG", statusRGHandler)
http.HandleFunc("/statusCG", statusCGHandler)
http.HandleFunc("/statusWG", statusWGHandler)
http.HandleFunc("/statusNG", statusNGHandler)
http.HandleFunc("/summary", summaryHandler)
http.HandleFunc("/", emptyHandler)
// listen only on localhost
err := http.ListenAndServe("127.0.0.1:"+port, nil)
@ -28,18 +29,30 @@ func startHTTP(port string) { @@ -28,18 +29,30 @@ func startHTTP(port string) {
}
// reflectHandler processes all requests and returns output in the requested format
func dnsHandler(w http.ResponseWriter, r *http.Request) {
// dnsWebHandler processes all requests and returns output in the requested format
func dnsWebHandler(w http.ResponseWriter, r *http.Request) {
st := time.Now()
// skip the s= from the raw query
n := r.FormValue("s")
s := getSeederByName(n)
if s == nil {
writeHeader(w, r)
fmt.Fprintf(w, "No seeder found: %s", html.EscapeString(n))
writeFooter(w, r, st)
return
}
// FIXME - This is ugly code and needs to be cleaned up a lot
// get v4 std addresses
v4std := getv4stdRR()
v4non := getv4nonRR()
v6std := getv6stdRR()
v6non := getv6nonRR()
config.dnsmtx.RLock()
// if the dns map does not have a key for the request it will return an empty slice
v4std := config.dns[s.dnsHost+".A"]
v4non := config.dns["nonstd."+s.dnsHost+".A"]
v6std := config.dns[s.dnsHost+".AAAA"]
v6non := config.dns["nonstd."+s.dnsHost+".AAAA"]
config.dnsmtx.RUnlock()
var v4stdstr, v4nonstr []string
var v6stdstr, v6nonstr []string
@ -174,26 +187,37 @@ func statusNGHandler(w http.ResponseWriter, r *http.Request) { @@ -174,26 +187,37 @@ func statusNGHandler(w http.ResponseWriter, r *http.Request) {
type webstatus struct {
Key string
Value string
Seeder string
}
func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
startT := time.Now()
// read the seeder name
n := r.FormValue("s")
s := getSeederByName(n)
if s == nil {
writeHeader(w, r)
fmt.Fprintf(w, "No seeder found called %s", html.EscapeString(n))
writeFooter(w, r, startT)
return
}
// gather all the info before writing anything to the remote browser
ws := generateWebStatus(status)
ws := generateWebStatus(s, status)
st := `
<center>
<table border=1>
<tr>
<th>Twistee</th>
<th>Node</th>
<th>Summary</th>
</tr>
{{range .}}
<tr>
<td>
<a href="/twistee?tw={{.Key}}">{{.Key}}</a>
<a href="/node?s={{.Seeder}}&nd={{.Key}}">{{.Key}}</a>
</td>
<td>
{{.Value}}
@ -207,18 +231,18 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { @@ -207,18 +231,18 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
writeHeader(w, r)
if len(ws) == 0 {
fmt.Fprintf(w, "No Twistees found with this status")
fmt.Fprintf(w, "No Nodes found with this status")
} else {
switch status {
case statusRG:
fmt.Fprintf(w, "<center><b>Twistee Status: statusRG - (Reported Good) Have not been able to get addresses yet</b></center>")
fmt.Fprintf(w, "<center><b>Node Status: statusRG - (Reported Good) Have not been able to get addresses yet</b></center>")
case statusCG:
fmt.Fprintf(w, "<center><b>Twistee Status: statusCG - (Currently Good) Able to connect and get addresses</b></center>")
fmt.Fprintf(w, "<center><b>Node Status: statusCG - (Currently Good) Able to connect and get addresses</b></center>")
case statusWG:
fmt.Fprintf(w, "<center><b>Twistee Status: statusWG - (Was Good) Was Ok but now can not get addresses</b></center>")
fmt.Fprintf(w, "<center><b>Node Status: statusWG - (Was Good) Was Ok but now can not get addresses</b></center>")
case statusNG:
fmt.Fprintf(w, "<center><b>Twistee Status: statusNG - (No Good) Unable to get addresses</b></center>")
fmt.Fprintf(w, "<center><b>Node Status: statusNG - (No Good) Unable to get addresses</b></center>")
}
t := template.New("Status template")
t, err := t.Parse(st)
@ -234,7 +258,59 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { @@ -234,7 +258,59 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
writeFooter(w, r, startT)
}
// copy Twistee details into a template friendly struct
// generateWebStatus is given a node status and returns a slice of webstatus structures
// ready to be ranged over by an html/template
func generateWebStatus(s *dnsseeder, status uint32) (ws []webstatus) {
s.mtx.RLock()
defer s.mtx.RUnlock()
var valueStr string
for k, v := range s.theList {
if v.status != status {
continue
}
switch status {
case statusRG:
valueStr = fmt.Sprintf("<b>Fail Count:</b> %v <b>DNS Type:</b> %s",
v.connectFails,
v.dns2str())
case statusCG:
valueStr = fmt.Sprintf("<b>Remote Version:</b> %v%s <b>Last Block:</b> %v <b>DNS Type:</b> %s",
v.version,
v.strVersion,
v.lastBlock,
v.dns2str())
case statusWG:
valueStr = fmt.Sprintf("<b>Last Try:</b> %s ago <b>Last Status:</b> %s\n",
time.Since(v.lastTry).String(),
v.statusStr)
case statusNG:
valueStr = fmt.Sprintf("<b>Fail Count:</b> %v <b>Last Try:</b> %s ago <b>Last Status:</b> %s\n",
v.connectFails,
time.Since(v.lastTry).String(),
v.statusStr)
default:
valueStr = ""
}
ows := webstatus{
Key: k,
Value: valueStr,
Seeder: s.name,
}
ws = append(ws, ows)
}
return ws
}
// copy Node details into a template friendly struct
type webtemplate struct {
Key string
IP string
@ -257,16 +333,16 @@ type webtemplate struct { @@ -257,16 +333,16 @@ type webtemplate struct {
Nonstdip string
}
// reflectHandler processes all requests and returns output in the requested format
func twisteeHandler(w http.ResponseWriter, r *http.Request) {
// nodeHandler displays details about one node
func nodeHandler(w http.ResponseWriter, r *http.Request) {
st := time.Now()
twt := `
ndt := `
<center>
<table border=1>
<tr>
<th>Twistee {{.Key}}</th><th>Details</th>
<th>Node {{.Key}}</th><th>Details</th>
</tr>
<tr><td>IP Address</td><td>{{.IP}}</td></tr>
<tr><td>Port</td><td>{{.Port}}</td></tr>
@ -285,111 +361,68 @@ func twisteeHandler(w http.ResponseWriter, r *http.Request) { @@ -285,111 +361,68 @@ func twisteeHandler(w http.ResponseWriter, r *http.Request) {
</table>
</center>
`
s := config.seeder
// read the seeder name
n := r.FormValue("s")
s := getSeederByName(n)
if s == nil {
writeHeader(w, r)
fmt.Fprintf(w, "No seeder found called %s", html.EscapeString(n))
writeFooter(w, r, st)
return
}
s.mtx.RLock()
defer s.mtx.RUnlock()
// skip the tw= from the raw query
k := html.UnescapeString(r.URL.RawQuery[3:])
k := r.FormValue("nd")
writeHeader(w, r)
if _, ok := s.theList[k]; ok == false {
fmt.Fprintf(w, "Sorry there is no Twistee with those details\n")
fmt.Fprintf(w, "Sorry there is no Node with those details\n")
} else {
tw := s.theList[k]
nd := s.theList[k]
wt := webtemplate{
IP: tw.na.IP.String(),
Port: tw.na.Port,
Dnstype: tw.dns2str(),
Nonstdip: tw.nonstdIP.String(),
Statusstr: tw.statusStr,
Lastconnect: tw.lastConnect.String(),
Lastconnectago: time.Since(tw.lastConnect).String(),
Lasttry: tw.lastTry.String(),
Lasttryago: time.Since(tw.lastTry).String(),
Crawlstart: tw.crawlStart.String(),
Crawlstartago: time.Since(tw.crawlStart).String(),
Connectfails: tw.connectFails,
Crawlactive: tw.crawlActive,
Version: tw.version,
Strversion: tw.strVersion,
Services: tw.services.String(),
Lastblock: tw.lastBlock,
}
// display details for the Twistee
t := template.New("Twistee template")
t, err := t.Parse(twt)
IP: nd.na.IP.String(),
Port: nd.na.Port,
Dnstype: nd.dns2str(),
Nonstdip: nd.nonstdIP.String(),
Statusstr: nd.statusStr,
Lastconnect: nd.lastConnect.String(),
Lastconnectago: time.Since(nd.lastConnect).String(),
Lasttry: nd.lastTry.String(),
Lasttryago: time.Since(nd.lastTry).String(),
Crawlstart: nd.crawlStart.String(),
Crawlstartago: time.Since(nd.crawlStart).String(),
Connectfails: nd.connectFails,
Crawlactive: nd.crawlActive,
Version: nd.version,
Strversion: nd.strVersion,
Services: nd.services.String(),
Lastblock: nd.lastBlock,
}
// display details for the Node
t := template.New("Node template")
t, err := t.Parse(ndt)
if err != nil {
log.Printf("error parsing Twistee template %v\n", err)
log.Printf("error parsing Node template %v\n", err)
}
err = t.Execute(w, wt)
if err != nil {
log.Printf("error executing Twistee template %v\n", err)
log.Printf("error executing Node template %v\n", err)
}
}
writeFooter(w, r, st)
}
// generateWebStatus is given a twistee status and returns a slice of webstatus structures
// ready to be ranged over by an html/template
func generateWebStatus(status uint32) (ws []webstatus) {
s := config.seeder
s.mtx.RLock()
defer s.mtx.RUnlock()
var valueStr string
// summaryHandler displays details about one node
func summaryHandler(w http.ResponseWriter, r *http.Request) {
for k, v := range s.theList {
if v.status != status {
continue
}
switch status {
case statusRG:
valueStr = fmt.Sprintf("<b>Fail Count:</b> %v <b>DNS Type:</b> %s",
v.connectFails,
v.dns2str())
case statusCG:
valueStr = fmt.Sprintf("<b>Remote Version:</b> %v%s <b>Last Block:</b> %v <b>DNS Type:</b> %s",
v.version,
v.strVersion,
v.lastBlock,
v.dns2str())
case statusWG:
valueStr = fmt.Sprintf("<b>Last Try:</b> %s ago <b>Last Status:</b> %s\n",
time.Since(v.lastTry).String(),
v.statusStr)
case statusNG:
valueStr = fmt.Sprintf("<b>Fail Count:</b> %v <b>Last Try:</b> %s ago <b>Last Status:</b> %s\n",
v.connectFails,
time.Since(v.lastTry).String(),
v.statusStr)
default:
valueStr = ""
}
ows := webstatus{
Key: k,
Value: valueStr,
}
ws = append(ws, ows)
}
return ws
}
// genHeader will output the standard header
func writeHeader(w http.ResponseWriter, r *http.Request) {
st := time.Now()
var hc struct {
Name string
RG uint32
RGS uint32
CG uint32
@ -406,65 +439,85 @@ func writeHeader(w http.ResponseWriter, r *http.Request) { @@ -406,65 +439,85 @@ func writeHeader(w http.ResponseWriter, r *http.Request) {
DNSTotal uint32
}
writeHeader(w, r)
// loop through each of the seeders
for _, s := range config.seeders {
hc.Name = s.name
// fill the structs so they can be displayed via the template
counts.mtx.RLock()
hc.RG = counts.TwStatus[statusRG]
hc.RGS = counts.TwStarts[statusRG]
hc.CG = counts.TwStatus[statusCG]
hc.CGS = counts.TwStarts[statusCG]
hc.WG = counts.TwStatus[statusWG]
hc.WGS = counts.TwStarts[statusWG]
hc.NG = counts.TwStatus[statusNG]
hc.NGS = counts.TwStarts[statusNG]
s.counts.mtx.RLock()
hc.RG = s.counts.NdStatus[statusRG]
hc.RGS = s.counts.NdStarts[statusRG]
hc.CG = s.counts.NdStatus[statusCG]
hc.CGS = s.counts.NdStarts[statusCG]
hc.WG = s.counts.NdStatus[statusWG]
hc.WGS = s.counts.NdStarts[statusWG]
hc.NG = s.counts.NdStatus[statusNG]
hc.NGS = s.counts.NdStarts[statusNG]
hc.Total = hc.RG + hc.CG + hc.WG + hc.NG
hc.V4Std = counts.DNSCounts[dnsV4Std]
hc.V4Non = counts.DNSCounts[dnsV4Non]
hc.V6Std = counts.DNSCounts[dnsV6Std]
hc.V6Non = counts.DNSCounts[dnsV6Non]
hc.V4Std = s.counts.DNSCounts[dnsV4Std]
hc.V4Non = s.counts.DNSCounts[dnsV4Non]
hc.V6Std = s.counts.DNSCounts[dnsV6Std]
hc.V6Non = s.counts.DNSCounts[dnsV6Non]
hc.DNSTotal = hc.V4Std + hc.V4Non + hc.V6Std + hc.V6Non
counts.mtx.RUnlock()
s.counts.mtx.RUnlock()
// we are using basic and simple html here. No fancy graphics or css
h := `
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html><head><title>dnsseeder</title></head><body>
sp := `
<b>Stats for seeder: {{.Name}}</b>
<center>
<a href="/statusRG">statusRG</a>
<a href="/statusCG">statusCG</a>
<a href="/statusWG">statusWG</a>
<a href="/statusNG">statusNG</a>
<a href="/dns">DNS</a>
<br>
<table><tr><td>
Twistee Stats (count/started)<br>
Node Stats (count/started)<br>
<table border=1><tr>
<td>RG: {{.RG}}/{{.RGS}}</td><td>CG: {{.CG}}/{{.CGS}}</td><td>WG: {{.WG}}/{{.WGS}}</td><td>NG: {{.NG}}/{{.NGS}}</td><td>Total: {{.Total}}</td>
<td><a href="/statusRG?s={{.Name}}">RG: {{.RG}}/{{.RGS}}</a></td>
<td><a href="/statusCG?s={{.Name}}">CG: {{.CG}}/{{.CGS}}</a></td>
<td><a href="/statusWG?s={{.Name}}">WG: {{.WG}}/{{.WGS}}</a></td>
<td><a href="/statusNG?s={{.Name}}">NG: {{.NG}}/{{.NGS}}</a></td>
<td>Total: {{.Total}}</td>
</tr></table>
</td><td>
DNS Requests<br>
<table border=1><tr>
<td>V4 Std: {{.V4Std}}</td><td>V4 Non: {{.V4Non}}</td><td>V6 Std: {{.V6Std}}</td><td>V6 Non: {{.V6Non}}</td><td>Total: {{.DNSTotal}}</td>
<td>V4 Std: {{.V4Std}}</td>
<td>V4 Non: {{.V4Non}}</td>
<td>V6 Std: {{.V6Std}}</td>
<td>V6 Non: {{.V6Non}}</td>
<td><a href="/dns?s={{.Name}}">Total: {{.DNSTotal}}</a></td>
</tr></table>
</td></tr></table>
</center>
<hr>
`
t := template.New("Header template")
t, err := t.Parse(h)
t, err := t.Parse(sp)
if err != nil {
log.Printf("error parsing template %v\n", err)
log.Printf("error parsing summary template %v\n", err)
}
err = t.Execute(w, hc)
if err != nil {
log.Printf("error executing template %v\n", err)
log.Printf("error executing summary template %v\n", err)
}
}
writeFooter(w, r, st)
}
// writeHeader will output the standard header
func writeHeader(w http.ResponseWriter, r *http.Request) {
// we are using basic and simple html here. No fancy graphics or css
h := `
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
<html><head><title>dnsseeder</title></head><body>
<center>
<a href="/summary">Summary</a>
</center>
<hr>
<br>
`
fmt.Fprintf(w, h)
}
// genFooter will output the standard footer
// writeFooter will output the standard footer
func writeFooter(w http.ResponseWriter, r *http.Request, st time.Time) {
// Footer needs to be exported for template processing to work
@ -483,7 +536,7 @@ func writeFooter(w http.ResponseWriter, r *http.Request, st time.Time) { @@ -483,7 +536,7 @@ func writeFooter(w http.ResponseWriter, r *http.Request, st time.Time) {
</center>
</body></html>
`
Footer.Uptime = time.Since(config.seeder.uptime).String()
Footer.Uptime = time.Since(config.uptime).String()
Footer.Version = config.version
Footer.Rt = time.Since(st).String()

181
main.go

@ -9,76 +9,86 @@ import ( @@ -9,76 +9,86 @@ import (
"log"
"os"
"os/signal"
"strings"
"sync"
"sync/atomic"
"syscall"
"time"
"github.com/miekg/dns"
)
// twCounts holds various statistics about the running system
type twCounts struct {
TwStatus []uint32
TwStarts []uint32
DNSCounts []uint32
mtx sync.RWMutex
// ndCounts holds various statistics about the running system
type NodeCounts struct {
NdStatus []uint32 // number of nodes at each of the 4 statuses - RG, CG, WG, NG
NdStarts []uint32 // number of crawles started last startcrawlers run
DNSCounts []uint32 // number of dns requests for each dns type - dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non
mtx sync.RWMutex // protect the structures
}
// configData holds information on the application
type configData struct {
host string
port string
http string
version string
verbose bool
debug bool
stats bool
seeder *dnsseeder
uptime time.Time // application start time
port string // port for the dns server to listen on
http string // port for the web server to listen on
version string // application version
verbose bool // verbose output cmdline option
debug bool // debug cmdline option
stats bool // stats cmdline option
seeders map[string]*dnsseeder // holds a pointer to all the current seeders
smtx sync.RWMutex // protect the seeders map
dns map[string][]dns.RR // holds details of all the currently served dns records
dnsmtx sync.RWMutex // protect the dns map
dnsUnknown uint64 // the number of dns requests for we are not configured to handle
}
var config configData
var counts twCounts
var nwname string
var netfile string
func main() {
var j bool
// FIXME - update with git hash during build
config.version = "0.6.0"
config.uptime = time.Now()
// initialize the stats counters
counts.TwStatus = make([]uint32, maxStatusTypes)
counts.TwStarts = make([]uint32, maxStatusTypes)
counts.DNSCounts = make([]uint32, maxDNSTypes)
flag.StringVar(&nwname, "net", "", "Preconfigured Network config")
flag.StringVar(&config.host, "h", "", "DNS host to serve")
flag.StringVar(&netfile, "netfile", "", "List of json config files to load")
flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on")
flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
flag.BoolVar(&j, "j", false, "Write network template file (dnsseeder.json) and exit")
flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
flag.BoolVar(&config.debug, "d", false, "Display debug output")
flag.BoolVar(&config.stats, "s", false, "Display stats output")
flag.Parse()
if config.host == "" {
fmt.Printf("error - no hostname provided\n")
os.Exit(1)
if j == true {
createNetFile()
fmt.Printf("Template file has been created\n")
os.Exit(0)
}
// configure the network options so we can start crawling
thenet := selectNetwork(nwname)
if thenet == nil {
fmt.Printf("Error - No valid network specified. Please add -net=<network> from one of the following:\n")
for _, n := range getNetworkNames() {
fmt.Printf("%s\n", n)
}
netwFiles := strings.Split(netfile, ",")
if len(netwFiles) == 0 {
fmt.Printf("Error - No filenames specified. Please add -net=<file[, file2]> to load these files\n")
os.Exit(1)
}
// init the seeder
config.seeder = &dnsseeder{}
config.seeder.theList = make(map[string]*twistee)
config.seeder.uptime = time.Now()
config.seeder.net = thenet
config.seeders = make(map[string]*dnsseeder)
config.dns = make(map[string][]dns.RR)
for _, nwFile := range netwFiles {
nnw, err := loadNetwork(nwFile)
if err != nil {
fmt.Printf("Error loading data from netfile %s - %v\n", nwFile, err)
os.Exit(1)
}
if nnw != nil {
// FIXME - lock this
config.seeders[nnw.name] = nnw
}
}
if config.debug == true {
config.verbose = true
@ -88,37 +98,38 @@ func main() { @@ -88,37 +98,38 @@ func main() {
config.stats = true
}
log.Printf("Starting dnsseeder system for host %s.\n", config.host)
if config.verbose == false {
log.Printf("status - Running in quiet mode with limited output produced\n")
} else {
log.Printf("status - system is configured for %s\n", config.seeder.net.name)
for _, v := range config.seeders {
log.Printf("status - system is configured for network: %s\n", v.name)
}
}
// start the web interface if we want it running
if config.http != "" {
go startHTTP(config.http)
}
// start dns server
dns.HandleFunc("nonstd."+config.host, handleDNSNon)
dns.HandleFunc(config.host, handleDNSStd)
dns.HandleFunc(".", handleDNS)
go serve("udp", config.port)
//go serve("tcp", config.port)
// seed the seeder with some ip addresses
config.seeder.initCrawlers()
// start first crawl
config.seeder.startCrawlers()
for _, s := range config.seeders {
s.initCrawlers()
s.startCrawlers()
}
sig := make(chan os.Signal)
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
// extract good dns records from all twistees on regular basis
// extract good dns records from all nodes on regular basis
dnsChan := time.NewTicker(time.Second * dnsDelay).C
// used to start crawlers on a regular basis
crawlChan := time.NewTicker(time.Second * crawlDelay).C
// used to remove old statusNG twistees that have reached fail count
// used to remove old statusNG nodes that have reached fail count
auditChan := time.NewTicker(time.Minute * auditDelay).C
dowhile := true
@ -128,40 +139,88 @@ func main() { @@ -128,40 +139,88 @@ func main() {
dowhile = false
case <-auditChan:
if config.debug {
log.Printf("debug - Audit twistees timer triggered\n")
log.Printf("debug - Audit nodes timer triggered\n")
}
for _, s := range config.seeders {
// FIXME goroutines for these
s.auditNodes()
}
config.seeder.auditClients()
case <-dnsChan:
if config.debug {
log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n")
}
config.seeder.loadDNS()
for _, s := range config.seeders {
s.loadDNS()
}
case <-crawlChan:
if config.debug {
log.Printf("debug - Start crawlers timer triggered\n")
}
config.seeder.startCrawlers()
for _, s := range config.seeders {
s.startCrawlers()
}
}
}
// FIXME - call dns server.Shutdown()
fmt.Printf("\nProgram exiting. Bye\n")
}
// updateTwCounts runs in a goroutine and updates the global stats with the lates
// updateNodeCounts runs in a goroutine and updates the global stats with the latest
// counts from a startCrawlers run
func updateTwCounts(status, total, started uint32) {
func updateNodeCounts(s *dnsseeder, status, total, started uint32) {
// update the stats counters
counts.mtx.Lock()
counts.TwStatus[status] = total
counts.TwStarts[status] = started
counts.mtx.Unlock()
s.counts.mtx.Lock()
s.counts.NdStatus[status] = total
s.counts.NdStarts[status] = started
s.counts.mtx.Unlock()
}
// updateDNSCounts runs in a goroutine and updates the global stats for the number of DNS requests
func updateDNSCounts(dnsType uint32) {
counts.mtx.Lock()
counts.DNSCounts[dnsType]++
counts.mtx.Unlock()
func updateDNSCounts(name, qtype string) {
var ndType uint32
var counted bool
nonstd := strings.HasPrefix(name, "nonstd.")
switch qtype {
case "A":
if nonstd {
ndType = dnsV4Non
} else {
ndType = dnsV4Std
}
case "AAAA":
if nonstd {
ndType = dnsV6Non
} else {
ndType = dnsV6Std
}
default:
ndType = dnsInvalid
}
// for DNS requests we do not have a reference to a seeder so we have to find it
for _, s := range config.seeders {
s.counts.mtx.Lock()
if name == s.dnsHost || name == "nonstd."+s.dnsHost {
s.counts.DNSCounts[ndType]++
counted = true
}
s.counts.mtx.Unlock()
}
if counted != true {
atomic.AddUint64(&config.dnsUnknown, 1)
}
}
func getSeederByName(name string) *dnsseeder {
for _, s := range config.seeders {
if s.name == name {
return s
}
}
return nil
}
/*

181
network.go

@ -1,73 +1,142 @@ @@ -1,73 +1,142 @@
package main
import (
"encoding/json"
"errors"
"fmt"
"github.com/btcsuite/btcd/wire"
"log"
"os"
"strconv"
)
// network struct holds config details for the network the seeder is using
type network struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this network uses
pver uint32 // minimum block height for the network
ttl uint32 // DNS TTL to use for this network
name string // Short name for the network
description string // Long description for the network
seeders []string // slice of seeders to pull ip addresses when starting this seeder
maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status
// JNetwork is the exported struct that is read from the network file
type JNetwork struct {
Name string
Desc string
Id string
Port uint16
Pver uint32
DNSName string
TTL uint32
Seeder1 string
Seeder2 string
Seeder3 string
}
// getNetworkNames returns a slice of the networks that have been configured
func getNetworkNames() []string {
return []string{"twister", "bitcoin", "bitcoin-testnet"}
func createNetFile() {
// create a standard json template file that can be loaded into the app
// create a struct to encode with json
jnw := &JNetwork{
Id: "0xabcdef01",
Port: 1234,
Pver: 70001,
TTL: 600,
DNSName: "seeder.example.com",
Name: "SeederNet",
Desc: "Description of SeederNet",
Seeder1: "seeder1.example.com",
Seeder2: "seed1.bob.com",
Seeder3: "seed2.example.com",
}
f, err := os.Create("dnsseeder.json")
if err != nil {
log.Printf("error creating template file: %v\n", err)
}
defer f.Close()
j, jerr := json.MarshalIndent(jnw, "", " ")
if jerr != nil {
log.Printf("error parsing json: %v\n", err)
}
_, ferr := f.Write(j)
if ferr != nil {
log.Printf("error writing to template file: %v\n", err)
}
}
// selectNetwork will return a network struct for a given network
func selectNetwork(name string) *network {
switch name {
case "twister":
return &network{
id: 0xd2bbdaf0,
port: 28333,
pver: 60000,
ttl: 600,
maxSize: 1000,
name: "TwisterNet",
description: "Twister P2P Net",
seeders: []string{"seed2.twister.net.co", "seed.twister.net.co", "seed3.twister.net.co"},
maxStart: []uint32{15, 15, 15, 30},
delay: []int64{184, 678, 237, 1876},
func loadNetwork(fName string) (*dnsseeder, error) {
nwFile, err := os.Open(fName)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error reading network file: %v", err))
}
case "bitcoin":
return &network{
id: 0xd9b4bef9,
port: 8333,
pver: 70001,
ttl: 900,
maxSize: 1250,
name: "BitcoinMainNet",
description: "Bitcoin Main Net",
seeders: []string{"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be"},
maxStart: []uint32{20, 20, 20, 30},
delay: []int64{210, 789, 234, 1876},
defer nwFile.Close()
var jnw JNetwork
jsonParser := json.NewDecoder(nwFile)
if err = jsonParser.Decode(&jnw); err != nil {
return nil, errors.New(fmt.Sprintf("Error decoding network file: %v", err))
}
case "bitcoin-testnet":
return &network{
id: 0xdab5bffa,
port: 18333,
pver: 70001,
ttl: 300,
maxSize: 250,
name: "BitcoinTestNet",
description: "Bitcoin Test Net",
seeders: []string{"testnet-seed.alexykot.me", "testnet-seed.bitcoin.petertodd.org", "testnet-seed.bluematt.me", "testnet-seed.bitcoin.schildbach.de"},
maxStart: []uint32{15, 15, 15, 30},
delay: []int64{184, 678, 237, 1876},
return initNetwork(jnw, jnw.Name)
}
func initNetwork(jnw JNetwork, name string) (*dnsseeder, error) {
if jnw.Port == 0 {
return nil, errors.New(fmt.Sprintf("Invalid port supplied: %v", jnw.Port))
}
default:
return nil
if jnw.DNSName == "" {
return nil, errors.New(fmt.Sprintf("No DNS Hostname supplied"))
}
if _, ok := config.seeders[jnw.Name]; ok {
return nil, errors.New(fmt.Sprintf("Name already exists from previous file - %s", jnw.Name))
}
// init the seeder
seeder := &dnsseeder{}
seeder.theList = make(map[string]*node)
seeder.port = jnw.Port
seeder.pver = jnw.Pver
seeder.ttl = jnw.TTL
seeder.name = jnw.Name
seeder.desc = jnw.Desc
seeder.dnsHost = jnw.DNSName
// conver the network magic number to a Uint32
t1, err := strconv.ParseUint(jnw.Id, 0, 32)
if err != nil {
return nil, errors.New(fmt.Sprintf("Error converting Network Magic number: %v", err))
}
seeder.id = wire.BitcoinNet(t1)
// load the seeder dns
seeder.seeders = make([]string, 3)
seeder.seeders[0] = jnw.Seeder1
seeder.seeders[1] = jnw.Seeder2
seeder.seeders[2] = jnw.Seeder3
// add some checks to the start & delay values to keep them sane
seeder.maxStart = []uint32{20, 20, 20, 30}
seeder.delay = []int64{210, 789, 234, 1876}
seeder.maxSize = 1250
// initialize the stats counters
seeder.counts.NdStatus = make([]uint32, maxStatusTypes)
seeder.counts.NdStarts = make([]uint32, maxStatusTypes)
seeder.counts.DNSCounts = make([]uint32, maxDNSTypes)
// some sanity checks on the loaded config options
if seeder.ttl < 60 {
seeder.ttl = 60
}
// check for duplicates
for _, v := range config.seeders {
if v.id == seeder.id {
return nil, errors.New(fmt.Sprintf("Duplicate Magic id. Already loaded for %s so can not be used for %s", v.id, v.name, seeder.name))
}
if v.dnsHost == seeder.dnsHost {
return nil, errors.New(fmt.Sprintf("Duplicate DNS names. Already loaded %s for %s so can not be used for %s", v.dnsHost, v.name, seeder.name))
}
}
return seeder, nil
}
/*

12
twistee.go → node.go

@ -7,8 +7,8 @@ import ( @@ -7,8 +7,8 @@ import (
"github.com/btcsuite/btcd/wire"
)
// Twistee struct contains details on one twister client
type twistee struct {
// Node struct contains details on one client
type node struct {
na *wire.NetAddress // holds ip address & port details
lastConnect time.Time // last time we sucessfully connected to this client
lastTry time.Time // last time we tried to connect to this client
@ -28,8 +28,8 @@ type twistee struct { @@ -28,8 +28,8 @@ type twistee struct {
}
// status2str will return the string description of the status
func (tw twistee) status2str() string {
switch tw.status {
func (nd node) status2str() string {
switch nd.status {
case statusRG:
return "statusRG"
case statusCG:
@ -44,8 +44,8 @@ func (tw twistee) status2str() string { @@ -44,8 +44,8 @@ func (tw twistee) status2str() string {
}
// dns2str will return the string description of the dns type
func (tw twistee) dns2str() string {
switch tw.dnsType {
func (nd node) dns2str() string {
switch nd.dnsType {
case dnsV4Std:
return "v4 standard port"
case dnsV4Non:

139
seeder.go

@ -22,9 +22,9 @@ const ( @@ -22,9 +22,9 @@ const (
auditDelay = 22 // minutes between audit channel ticks
dnsDelay = 57 // seconds between updates to active dns record list
maxFails = 58 // max number of connect fails before we delete a twistee. Just over 24 hours(checked every 33 minutes)
maxFails = 58 // max number of connect fails before we delete a node. Just over 24 hours(checked every 33 minutes)
maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout
maxTo = 250 // max seconds (4min 10 sec) for all comms to node to complete before we timeout
)
const (
@ -37,48 +37,73 @@ const ( @@ -37,48 +37,73 @@ const (
)
const (
// twistee status
statusRG = iota // reported good status. A remote twistee has reported this ip but we have not connected
statusCG // confirmed good. We have connected to the twistee and received addresses
statusWG // was good. Twistee was confirmed good but now having problems
// node status
statusRG = iota // reported good status. A remote node has reported this ip but we have not connected
statusCG // confirmed good. We have connected to the node and received addresses
statusWG // was good. node was confirmed good but now having problems
statusNG // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
maxStatusTypes // used in main to allocate slice
)
type dnsseeder struct {
net *network // network struct with config options for this network
uptime time.Time // as the name says
theList map[string]*twistee // the list of current clients
/* NdCounts holds various statistics about the running system
type NdCounts struct {
NdStatus []uint32
NdStarts []uint32
DNSCounts []uint32
mtx sync.RWMutex
}
*/
type dnsseeder struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
theList map[string]*node // the list of current nodes
mtx sync.RWMutex // protect thelist
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
dnsHost string // dns host we will serve results for this domain
name string // Short name for the network
desc string // Long description for the network
seeders []string // slice of seeders to pull ip addresses when starting this seeder
maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status
counts NodeCounts // structure to hold stats for this seeder
}
// initCrawlers needs to be run before the startCrawlers so it can get
// a list of current ip addresses from the other seeders and therefore
// start the crawl process
func (s *dnsseeder) initCrawlers() {
// get a list of permenant seeders
seeders := s.net.seeders
for _, aseeder := range seeders {
for _, aseeder := range s.seeders {
c := 0
if aseeder == "" {
continue
}
newRRs, err := net.LookupHost(aseeder)
if err != nil {
log.Printf("status - unable to do initial lookup to seeder %s %v\n", aseeder, err)
log.Printf("%s: unable to do initial lookup to seeder %s %v\n", s.name, aseeder, err)
continue
}
for _, ip := range newRRs {
if newIP := net.ParseIP(ip); newIP != nil {
// 1 at the end is the services flag
if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, s.net.port, 1)); x == true {
if x := s.addNa(wire.NewNetAddressIPPort(newIP, s.port, 1)); x == true {
c++
}
}
}
if config.verbose {
log.Printf("status - completed import of %v addresses from %s\n", c, aseeder)
log.Printf("%s: completed import of %v addresses from %s\n", s.name, c, aseeder)
}
}
if len(s.theList) == 0 {
log.Printf("%s: Error: No ip addresses from seeders so I have nothing to crawl.\n", s.name)
for _, v := range s.seeders {
log.Printf("%s: Seeder: %s\n", s.name, v)
}
}
}
@ -90,7 +115,7 @@ func (s *dnsseeder) startCrawlers() { @@ -90,7 +115,7 @@ func (s *dnsseeder) startCrawlers() {
tcount := len(s.theList)
if tcount == 0 {
if config.debug {
log.Printf("debug - startCrawlers fail: no twistees available\n")
log.Printf("debug - startCrawlers fail: no node ailable\n")
}
return
}
@ -104,10 +129,10 @@ func (s *dnsseeder) startCrawlers() { @@ -104,10 +129,10 @@ func (s *dnsseeder) startCrawlers() {
started uint32 // count of goroutines started for this type
delay int64 // number of second since last try
}{
{"statusRG", statusRG, s.net.maxStart[statusRG], 0, 0, s.net.delay[statusRG]},
{"statusCG", statusCG, s.net.maxStart[statusCG], 0, 0, s.net.delay[statusCG]},
{"statusWG", statusWG, s.net.maxStart[statusWG], 0, 0, s.net.delay[statusWG]},
{"statusNG", statusNG, s.net.maxStart[statusNG], 0, 0, s.net.delay[statusNG]},
{"statusRG", statusRG, s.maxStart[statusRG], 0, 0, s.delay[statusRG]},
{"statusCG", statusCG, s.maxStart[statusCG], 0, 0, s.delay[statusCG]},
{"statusWG", statusWG, s.maxStart[statusWG], 0, 0, s.delay[statusWG]},
{"statusNG", statusNG, s.maxStart[statusNG], 0, 0, s.delay[statusNG]},
}
s.mtx.RLock()
@ -118,16 +143,16 @@ func (s *dnsseeder) startCrawlers() { @@ -118,16 +143,16 @@ func (s *dnsseeder) startCrawlers() {
// range on a map will not return items in the same order each time
// so this is a random'ish selection
for _, tw := range s.theList {
for _, nd := range s.theList {
if tw.status != c.status {
if nd.status != c.status {
continue
}
// stats count
c.totalCount++
if tw.crawlActive == true {
if nd.crawlActive == true {
continue
}
@ -135,23 +160,23 @@ func (s *dnsseeder) startCrawlers() { @@ -135,23 +160,23 @@ func (s *dnsseeder) startCrawlers() {
continue
}
if (time.Now().Unix() - c.delay) <= tw.lastTry.Unix() {
if (time.Now().Unix() - c.delay) <= nd.lastTry.Unix() {
continue
}
// all looks good so start a go routine to crawl the remote twistee
go crawlTwistee(s, tw)
// all looks good so start a go routine to crawl the remote node
go crawlNode(s, nd)
c.started++
}
log.Printf("stats - started crawler: %s total: %v started: %v\n", c.desc, c.totalCount, c.started)
log.Printf("%s: started crawler: %s total: %v started: %v\n", s.name, c.desc, c.totalCount, c.started)
// update the global stats in another goroutine to free the main goroutine
// for other work
go updateTwCounts(c.status, c.totalCount, c.started)
go updateNodeCounts(s, c.status, c.totalCount, c.started)
}
log.Printf("stats - crawlers started. total twistees: %d\n", tcount)
log.Printf("%s: crawlers started. total clients: %d\n", s.name, tcount)
// returns and read lock released
}
@ -191,7 +216,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { @@ -191,7 +216,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
return false
}
nt := twistee{
nt := node{
na: nNa,
lastConnect: time.Now(),
version: 0,
@ -203,7 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { @@ -203,7 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
// select the dns type based on the remote address type and port
if x := nt.na.IP.To4(); x == nil {
// not ipv4
if nNa.Port != s.net.port {
if nNa.Port != s.port {
nt.dnsType = dnsV6Non
// produce the nonstdIP
@ -214,7 +239,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { @@ -214,7 +239,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
}
} else {
// ipv4
if nNa.Port != s.net.port {
if nNa.Port != s.port {
nt.dnsType = dnsV4Non
// force ipv4 address into a 4 byte buffer
@ -270,7 +295,7 @@ func crc16(bs []byte) uint16 { @@ -270,7 +295,7 @@ func crc16(bs []byte) uint16 {
return crc
}
func (s *dnsseeder) auditClients() {
func (s *dnsseeder) auditNodes() {
c := 0
@ -280,64 +305,64 @@ func (s *dnsseeder) auditClients() { @@ -280,64 +305,64 @@ func (s *dnsseeder) auditClients() {
// cgGoal is 75% of the max statusCG clients we can crawl with the current network delay & maxStart settings.
// This allows us to cycle statusCG users to keep the list fresh
cgGoal := int(float64(float64(s.net.delay[statusCG]/crawlDelay)*float64(s.net.maxStart[statusCG])) * 0.75)
cgGoal := int(float64(float64(s.delay[statusCG]/crawlDelay)*float64(s.maxStart[statusCG])) * 0.75)
cgCount := 0
log.Printf("status - Audit start. statusCG Goal: %v System Uptime: %s\n", cgGoal, time.Since(s.uptime).String())
log.Printf("%s: Audit start. statusCG Goal: %v System Uptime: %s\n", s.name, cgGoal, time.Since(config.uptime).String())
s.mtx.Lock()
defer s.mtx.Unlock()
for k, tw := range s.theList {
for k, nd := range s.theList {
if tw.crawlActive == true {
if time.Now().Unix()-tw.crawlStart.Unix() >= 300 {
if nd.crawlActive == true {
if time.Now().Unix()-nd.crawlStart.Unix() >= 300 {
log.Printf("warning - long running crawl > 5 minutes ====\n- %s status:rating:fails %v:%v:%v crawl start: %s last status: %s\n====\n",
k,
tw.status,
tw.rating,
tw.connectFails,
tw.crawlStart.String(),
tw.statusStr)
nd.status,
nd.rating,
nd.connectFails,
nd.crawlStart.String(),
nd.statusStr)
}
}
// Audit task is to remove clients that we have not been able to connect to
if tw.status == statusNG && tw.connectFails > maxFails {
// Audit task is to remove node that we have not been able to connect to
if nd.status == statusNG && nd.connectFails > maxFails {
if config.verbose {
log.Printf("status - purging twistee %s after %v failed connections\n", k, tw.connectFails)
log.Printf("%s: purging node %s after %v failed connections\n", s.name, k, nd.connectFails)
}
c++
// remove the map entry and mark the old twistee as
// remove the map entry and mark the old node as
// nil so garbage collector will remove it
s.theList[k] = nil
delete(s.theList, k)
}
// If seeder is full then remove old NG clients and fill up with possible new CG clients
if tw.status == statusNG && iAmFull {
if nd.status == statusNG && iAmFull {
if config.verbose {
log.Printf("status - seeder full purging twistee %s\n", k)
log.Printf("%s: seeder full purging node %s\n", s.name, k)
}
c++
// remove the map entry and mark the old twistee as
// remove the map entry and mark the old node as
// nil so garbage collector will remove it
s.theList[k] = nil
delete(s.theList, k)
}
// check if we need to purge statusCG to freshen the list
if tw.status == statusCG {
if nd.status == statusCG {
if cgCount++; cgCount > cgGoal {
// we have enough statusCG clients so purge remaining to cycle through the list
if config.verbose {
log.Printf("status - seeder cycle statusCG - purging client %s\n", k)
log.Printf("%s: seeder cycle statusCG - purging node %s\n", s.name, k)
}
c++
// remove the map entry and mark the old twistee as
// remove the map entry and mark the old node as
// nil so garbage collector will remove it
s.theList[k] = nil
delete(s.theList, k)
@ -347,7 +372,7 @@ func (s *dnsseeder) auditClients() { @@ -347,7 +372,7 @@ func (s *dnsseeder) auditClients() {
}
if config.verbose {
log.Printf("status - Audit complete. %v twistees purged\n", c)
log.Printf("%s: Audit complete. %v nodes purged\n", s.name, c)
}
}
@ -359,7 +384,7 @@ func (s *dnsseeder) loadDNS() { @@ -359,7 +384,7 @@ func (s *dnsseeder) loadDNS() {
// isFull returns true if the number of remote clients is more than we want to store
func (s *dnsseeder) isFull() bool {
if len(s.theList) > s.net.maxSize {
if len(s.theList) > s.maxSize {
return true
}
return false

Loading…
Cancel
Save