From 9b5240c03b3a1d1af68c342360a4ed6692e4f79a Mon Sep 17 00:00:00 2001 From: Lyndsay Roger Date: Tue, 8 Sep 2015 10:03:44 +1200 Subject: [PATCH] Major refactor of the codebase to support multiple networks --- README.md | 25 ++- configs/bitcoin-test.json | 12 ++ configs/bitcoin.json | 12 ++ configs/dnsseeder.json | 12 ++ configs/twister.json | 12 ++ crawler.go | 134 ++++++++------- dns.go | 143 +++++---------- http.go | 353 ++++++++++++++++++++++---------------- main.go | 181 ++++++++++++------- network.go | 183 ++++++++++++++------ twistee.go => node.go | 12 +- seeder.go | 139 +++++++++------ 12 files changed, 708 insertions(+), 510 deletions(-) create mode 100644 configs/bitcoin-test.json create mode 100644 configs/bitcoin.json create mode 100644 configs/dnsseeder.json create mode 100644 configs/twister.json rename twistee.go => node.go (90%) diff --git a/README.md b/README.md index 2f95108..b7d3b89 100644 --- a/README.md +++ b/README.md @@ -1,9 +1,11 @@ # dnsseeder -Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/) +Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/) and the Bitcoin networks. -It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder +It is based on the original c++ seeders created for the Bitcoin network and copied to other similar networks. -This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option. +This application can seed one or more networks on the same ip address. At the moment there are config files for the Twister and Bitcoin networks. You use the -netfile commandline option to specify one or more comma seperated filenames to load the network configuration for that network. + +You can use the -j option to produce a sample json network config file (dnsseeder.json) in the current directory and then edit the file to seed your own network. Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/) @@ -34,19 +36,19 @@ The binary will then be available in ${HOME}/go/bin ## Usage - $ dnsseeder -h -net + $ dnsseeder -v -netfile An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running. -If you want to be able to view the web interface then add -w port for the web server to listen on. If this is not provided then no web interface will be available. With the web site running you can then access the site by http://localhost:port/statusCG +If you want to be able to view the web interface then add -w port for the web server to listen on. If this is not provided then no web interface will be available. With the web site running you can then access the site by http://localhost:port/summary **NOTE -** For security reasons the web server will only listen on localhost so you will need to either use an ssh tunnel or proxy requests via a web server like Nginx or Apache. ``` Command line Options: --h hostname to serve --net The network to seed. Currently twister, bitcoin, bitcoin-test +-netfile comma seperated list of json network config files to load +-j write a sample network config file in json format and exit. -p port to listen on for DNS requests -d Produce debug output -v Produce verbose output @@ -66,18 +68,11 @@ mkdir -p ${LOGDIR} gzip ${LOGDIR}/*.log -# pass through the logging level needed -if [ -z ${1} ]; then - THENET="twister" -else - THENET="${1}" -fi - cd echo echo "======= Run the Go Language dnsseed =======" echo -${HOME}/go/bin/dnsseeder -h -p ${THENET} -v -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log +${HOME}/go/bin/dnsseeder -p -v -w 8880 -netfile ${1} 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log ``` diff --git a/configs/bitcoin-test.json b/configs/bitcoin-test.json new file mode 100644 index 0000000..3bde5e0 --- /dev/null +++ b/configs/bitcoin-test.json @@ -0,0 +1,12 @@ +{ + "Name": "BitcoinNet-Test", + "Desc": "Bitcoin Test Net", + "Id": "0xdab5bffa", + "Port": 18333, + "Pver": 70001, + "DNSName": "btctseed.zagbot.com", + "TTL": 300, + "Seeder1": "testnet-seed.alexykot.me", + "Seeder2": "testnet-seed.bitcoin.petertodd.org", + "Seeder3": "testnet-seed.bluematt.me" +} diff --git a/configs/bitcoin.json b/configs/bitcoin.json new file mode 100644 index 0000000..9102659 --- /dev/null +++ b/configs/bitcoin.json @@ -0,0 +1,12 @@ +{ + "Name": "BitcoinNet", + "Desc": "Bitcoin Main Net", + "Id": "0xd9b4bef9", + "Port": 8333, + "Pver": 70001, + "DNSName": "btcseed.zagbot.com", + "TTL": 600, + "Seeder1": "dnsseed.bluematt.me", + "Seeder2": "bitseed.xf2.org", + "Seeder3": "dnsseed.bitcoin.dashjr.org" +} diff --git a/configs/dnsseeder.json b/configs/dnsseeder.json new file mode 100644 index 0000000..ec9609e --- /dev/null +++ b/configs/dnsseeder.json @@ -0,0 +1,12 @@ +{ + "Name": "SeederNet", + "Desc": "Description of SeederNet", + "Id": "0xabcdef01", + "Port": 1234, + "Pver": 70001, + "DNSName": "seeder.example.com", + "TTL": 600, + "Seeder1": "seeder1.example.com", + "Seeder2": "seed1.bob.com", + "Seeder3": "seed2.example.com" +} \ No newline at end of file diff --git a/configs/twister.json b/configs/twister.json new file mode 100644 index 0000000..0c72997 --- /dev/null +++ b/configs/twister.json @@ -0,0 +1,12 @@ +{ + "Name": "TwisterNet", + "Desc": "Peer to Peer Microblogging Network", + "Id": "0xd2bbdaf0", + "Port": 28333, + "Pver": 60000, + "DNSName": "dnsseed.zagbot.com", + "TTL": 600, + "Seeder1": "seed2.twister.net.co", + "Seeder2": "seed.twister.net.co", + "Seeder3": "seed3.twister.net.co" +} diff --git a/crawler.go b/crawler.go index cab44f3..c4ba931 100644 --- a/crawler.go +++ b/crawler.go @@ -20,82 +20,83 @@ func (e *crawlError) Error() string { return "err: " + e.errLoc + ": " + e.Err.Error() } -// crawlTwistee runs in a goroutine, crawls the remote ip and updates the master +// crawlNode runs in a goroutine, crawls the remote ip and updates the master // list of currently active addresses -func crawlTwistee(s *dnsseeder, tw *twistee) { +func crawlNode(s *dnsseeder, nd *node) { - tw.crawlActive = true - tw.crawlStart = time.Now() + nd.crawlActive = true + nd.crawlStart = time.Now() - defer crawlEnd(tw) + defer crawlEnd(nd) if config.debug { - log.Printf("debug - start crawl: twistee %s status: %v:%v lastcrawl: %s\n", - net.JoinHostPort(tw.na.IP.String(), - strconv.Itoa(int(tw.na.Port))), - tw.status, - tw.rating, - time.Since(tw.crawlStart).String()) + log.Printf("debug - start crawl: node %s status: %v:%v lastcrawl: %s\n", + net.JoinHostPort(nd.na.IP.String(), + strconv.Itoa(int(nd.na.Port))), + nd.status, + nd.rating, + time.Since(nd.crawlStart).String()) } // connect to the remote ip and ask them for their addr list - rna, e := crawlIP(s.net.pver, s.net.id, tw) + rna, e := crawlIP(s.pver, s.id, nd) if e != nil { - // update the fact that we have not connected to this twistee - tw.lastTry = time.Now() - tw.connectFails++ - tw.statusStr = e.Error() + // update the fact that we have not connected to this node + nd.lastTry = time.Now() + nd.connectFails++ + nd.statusStr = e.Error() - // update the status of this failed twistee - switch tw.status { + // update the status of this failed node + switch nd.status { case statusRG: // if we are full then any RG failures will skip directly to NG if s.isFull() { - tw.status = statusNG // not able to connect to this twistee so ignore - tw.statusTime = time.Now() + nd.status = statusNG // not able to connect to this node so ignore + nd.statusTime = time.Now() } else { - if tw.rating += 25; tw.rating > 30 { - tw.status = statusWG - tw.statusTime = time.Now() + if nd.rating += 25; nd.rating > 30 { + nd.status = statusWG + nd.statusTime = time.Now() } } case statusCG: - if tw.rating += 25; tw.rating >= 50 { - tw.status = statusWG - tw.statusTime = time.Now() + if nd.rating += 25; nd.rating >= 50 { + nd.status = statusWG + nd.statusTime = time.Now() } case statusWG: - if tw.rating += 15; tw.rating >= 100 { - tw.status = statusNG // not able to connect to this twistee so ignore - tw.statusTime = time.Now() + if nd.rating += 15; nd.rating >= 100 { + nd.status = statusNG // not able to connect to this node so ignore + nd.statusTime = time.Now() } } // no more to do so return which will shutdown the goroutine & call // the deffered cleanup if config.verbose { - log.Printf("status - failed crawl: twistee %s s:r:f: %v:%v:%v %s\n", - net.JoinHostPort(tw.na.IP.String(), - strconv.Itoa(int(tw.na.Port))), - tw.status, - tw.rating, - tw.connectFails, - tw.statusStr) + log.Printf("%s: failed crawl node: %s s:r:f: %v:%v:%v %s\n", + s.name, + net.JoinHostPort(nd.na.IP.String(), + strconv.Itoa(int(nd.na.Port))), + nd.status, + nd.rating, + nd.connectFails, + nd.statusStr) } return } // succesful connection and addresses received so mark status - if tw.status != statusCG { - tw.status = statusCG - tw.statusTime = time.Now() + if nd.status != statusCG { + nd.status = statusCG + nd.statusTime = time.Now() } - cs := tw.lastConnect - tw.rating = 0 - tw.connectFails = 0 - tw.lastConnect = time.Now() - tw.lastTry = time.Now() - tw.statusStr = "ok: received remote address list" + cs := nd.lastConnect + nd.rating = 0 + nd.connectFails = 0 + nd.lastConnect = time.Now() + nd.lastTry = time.Now() + nd.statusStr = "ok: received remote address list" added := 0 @@ -111,15 +112,16 @@ func crawlTwistee(s *dnsseeder, tw *twistee) { } if config.verbose { - log.Printf("status - crawl done: twistee: %s s:r:f: %v:%v:%v addr: %v:%v CrawlTime: %s Last connect: %v ago\n", - net.JoinHostPort(tw.na.IP.String(), - strconv.Itoa(int(tw.na.Port))), - tw.status, - tw.rating, - tw.connectFails, + log.Printf("%s: crawl done: node: %s s:r:f: %v:%v:%v addr: %v:%v CrawlTime: %s Last connect: %v ago\n", + s.name, + net.JoinHostPort(nd.na.IP.String(), + strconv.Itoa(int(nd.na.Port))), + nd.status, + nd.rating, + nd.connectFails, len(rna), added, - time.Since(tw.crawlStart).String(), + time.Since(nd.crawlStart).String(), time.Since(cs).String()) } @@ -127,17 +129,17 @@ func crawlTwistee(s *dnsseeder, tw *twistee) { } // crawlEnd is a deffered func to update theList after a crawl is all done -func crawlEnd(tw *twistee) { - tw.crawlActive = false - // FIXME - scan for long term crawl active twistees. Dial timeout is 10 seconds +func crawlEnd(nd *node) { + nd.crawlActive = false + // FIXME - scan for long term crawl active node. Dial timeout is 10 seconds // so should be done in under 5 minutes } // crawlIP retrievs a slice of ip addresses from a client -func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) { +func crawlIP(pver uint32, netID wire.BitcoinNet, nd *node) ([]*wire.NetAddress, *crawlError) { - ip := tw.na.IP.String() - port := strconv.Itoa(int(tw.na.Port)) + ip := nd.na.IP.String() + port := strconv.Itoa(int(nd.na.Port)) // get correct formatting for ipv6 addresses dialString := net.JoinHostPort(ip, port) @@ -151,7 +153,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres defer conn.Close() if config.debug { - log.Printf("debug - Connected to remote address: %s Last connect was %v ago\n", ip, time.Since(tw.lastConnect).String()) + log.Printf("debug - Connected to remote address: %s Last connect was %v ago\n", ip, time.Since(nd.lastConnect).String()) } // set a deadline for all comms to be done by. After this all i/o will error @@ -183,14 +185,14 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres if config.debug { log.Printf("%s - Remote version: %v\n", ip, msg.ProtocolVersion) } - // fill the Twistee struct with the remote details - tw.version = msg.ProtocolVersion - tw.services = msg.Services - tw.lastBlock = msg.LastBlock - if tw.strVersion != msg.UserAgent { + // fill the node struct with the remote details + nd.version = msg.ProtocolVersion + nd.services = msg.Services + nd.lastBlock = msg.LastBlock + if nd.strVersion != msg.UserAgent { // if the srtVersion is already the same then don't overwrite it. // saves the GC having to cleanup a perfectly good string - tw.strVersion = msg.UserAgent + nd.strVersion = msg.UserAgent } default: return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")} @@ -231,7 +233,7 @@ func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddres dowhile := true for dowhile == true { - // Using the Bitcoin lib for the Twister Net means it does not understand some + // Using the Bitcoin lib for the some networks means it does not understand some // of the commands and will error. We can ignore these as we are only // interested in the addr message and its content. msgaddr, _, _ := wire.ReadMessage(conn, pver, netID) diff --git a/dns.go b/dns.go index 11612bb..3081e5d 100644 --- a/dns.go +++ b/dns.go @@ -2,29 +2,11 @@ package main import ( "log" - "sync" + // "sync" "github.com/miekg/dns" ) -type currentIPs struct { - ipv4std []dns.RR - ipv4non []dns.RR - ipv6std []dns.RR - ipv6non []dns.RR - mtx sync.RWMutex -} - -// latest holds the slices of current ip addresses -var latest currentIPs - -// getLatestaRR returns a pointer to the latest slice of current dns.RR type -// dns.A records to pass back to the remote client -func getv4stdRR() []dns.RR { return latest.ipv4std } -func getv4nonRR() []dns.RR { return latest.ipv4non } -func getv6stdRR() []dns.RR { return latest.ipv6std } -func getv6nonRR() []dns.RR { return latest.ipv6non } - // updateDNS updates the current slices of dns.RR so incoming requests get a // fast answer func updateDNS(s *dnsseeder) { @@ -35,60 +17,61 @@ func updateDNS(s *dnsseeder) { // loop over each dns recprd type we need for t := range []int{dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non} { + // FIXME above needs to be convertwd into one scan of theList if possible numRR := 0 - for _, tw := range s.theList { + for _, nd := range s.theList { // when we reach max exit if numRR >= 25 { break } - if tw.status != statusCG { + if nd.status != statusCG { continue } if t == dnsV4Std || t == dnsV4Non { - if t == dnsV4Std && tw.dnsType == dnsV4Std { + if t == dnsV4Std && nd.dnsType == dnsV4Std { r := new(dns.A) - r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.A = tw.na.IP + r.Hdr = dns.RR_Header{Name: s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl} + r.A = nd.na.IP rr4std = append(rr4std, r) numRR++ } - // if the twistee is using a non standard port then add the encoded port info to DNS - if t == dnsV4Non && tw.dnsType == dnsV4Non { + // if the node is using a non standard port then add the encoded port info to DNS + if t == dnsV4Non && nd.dnsType == dnsV4Non { r := new(dns.A) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.A = tw.na.IP + r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl} + r.A = nd.na.IP rr4non = append(rr4non, r) numRR++ r = new(dns.A) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.A = tw.nonstdIP + r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.ttl} + r.A = nd.nonstdIP rr4non = append(rr4non, r) numRR++ } } if t == dnsV6Std || t == dnsV6Non { - if t == dnsV6Std && tw.dnsType == dnsV6Std { + if t == dnsV6Std && nd.dnsType == dnsV6Std { r := new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.AAAA = tw.na.IP + r.Hdr = dns.RR_Header{Name: s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl} + r.AAAA = nd.na.IP rr6std = append(rr6std, r) numRR++ } - // if the twistee is using a non standard port then add the encoded port info to DNS - if t == dnsV6Non && tw.dnsType == dnsV6Non { + // if the node is using a non standard port then add the encoded port info to DNS + if t == dnsV6Non && nd.dnsType == dnsV6Non { r := new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.AAAA = tw.na.IP + r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl} + r.AAAA = nd.na.IP rr6non = append(rr6non, r) numRR++ r = new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} - r.AAAA = tw.nonstdIP + r.Hdr = dns.RR_Header{Name: "nonstd." + s.dnsHost + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.ttl} + r.AAAA = nd.nonstdIP rr6non = append(rr6non, r) numRR++ } @@ -100,32 +83,32 @@ func updateDNS(s *dnsseeder) { s.mtx.RUnlock() - latest.mtx.Lock() + config.dnsmtx.Lock() + // update the map holding the details for this seeder for t := range []int{dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non} { switch t { case dnsV4Std: - latest.ipv4std = rr4std + config.dns[s.dnsHost+".A"] = rr4std case dnsV4Non: - latest.ipv4non = rr4non + config.dns["nonstd."+s.dnsHost+".A"] = rr4non case dnsV6Std: - latest.ipv6std = rr6std + config.dns[s.dnsHost+".AAAA"] = rr6std case dnsV6Non: - latest.ipv6non = rr6non + config.dns["nonstd."+s.dnsHost+".AAAA"] = rr6non } } - latest.mtx.Unlock() + config.dnsmtx.Unlock() if config.debug { log.Printf("debug - DNS update complete - rr4std: %v rr4non: %v rr6std: %v rr6non: %v\n", len(rr4std), len(rr4non), len(rr6std), len(rr6non)) } } -// handleDNSStd processes a DNS request from remote client and returns +// handleDNS processes a DNS request from remote client and returns // a list of current ip addresses that the crawlers consider current. -// This function returns addresses that use the standard port -func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) { +func handleDNS(w dns.ResponseWriter, r *dns.Msg) { m := &dns.Msg{MsgHdr: dns.MsgHdr{ Authoritative: true, @@ -137,67 +120,31 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) { switch r.Question[0].Qtype { case dns.TypeA: - latest.mtx.RLock() - m.Answer = getv4stdRR() - latest.mtx.RUnlock() qtype = "A" - // start a goroutine to update the global counters then get back to answering this request - go updateDNSCounts(dnsV4Std) case dns.TypeAAAA: - latest.mtx.RLock() - m.Answer = getv6stdRR() - latest.mtx.RUnlock() qtype = "AAAA" - go updateDNSCounts(dnsV6Std) + case dns.TypeTXT: + qtype = "TXT" + case dns.TypeMX: + qtype = "MX" + case dns.TypeNS: + qtype = "NS" default: - // return no answer to all other queries - - } - - w.WriteMsg(m) - - if config.debug { - log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) + qtype = "UNKNOWN" } -} -// handleDNSNon processes a DNS request from remote client and returns -// a list of current ip addresses that the crawlers consider current. -// This function returns addresses that use the non standard port -func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) { - - m := &dns.Msg{MsgHdr: dns.MsgHdr{ - Authoritative: true, - RecursionAvailable: false, - }} - m.SetReply(r) - - var qtype string - - switch r.Question[0].Qtype { - case dns.TypeA: - latest.mtx.RLock() - m.Answer = getv4nonRR() - latest.mtx.RUnlock() - qtype = "A" - // start a goroutine to update the global counters then get back to answering this request - go updateDNSCounts(dnsV4Non) - case dns.TypeAAAA: - latest.mtx.RLock() - m.Answer = getv6nonRR() - latest.mtx.RUnlock() - qtype = "AAAA" - go updateDNSCounts(dnsV6Non) - default: - // return no answer to all other queries - - } + config.dnsmtx.RLock() + // if the dns map does not have a key for the request it will return an empty slice + m.Answer = config.dns[r.Question[0].Name+qtype] + config.dnsmtx.RUnlock() w.WriteMsg(m) if config.debug { - log.Printf("debug - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) + log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) } + // update the stats in a goroutine + go updateDNSCounts(r.Question[0].Name, qtype) } // serve starts the requested DNS server listening on the requested port diff --git a/http.go b/http.go index 2cc4939..7e89ac9 100644 --- a/http.go +++ b/http.go @@ -13,12 +13,13 @@ import ( // to the dnsseeder func startHTTP(port string) { - http.HandleFunc("/dns", dnsHandler) - http.HandleFunc("/twistee", twisteeHandler) + http.HandleFunc("/dns", dnsWebHandler) + http.HandleFunc("/node", nodeHandler) http.HandleFunc("/statusRG", statusRGHandler) http.HandleFunc("/statusCG", statusCGHandler) http.HandleFunc("/statusWG", statusWGHandler) http.HandleFunc("/statusNG", statusNGHandler) + http.HandleFunc("/summary", summaryHandler) http.HandleFunc("/", emptyHandler) // listen only on localhost err := http.ListenAndServe("127.0.0.1:"+port, nil) @@ -28,18 +29,30 @@ func startHTTP(port string) { } -// reflectHandler processes all requests and returns output in the requested format -func dnsHandler(w http.ResponseWriter, r *http.Request) { +// dnsWebHandler processes all requests and returns output in the requested format +func dnsWebHandler(w http.ResponseWriter, r *http.Request) { st := time.Now() + // skip the s= from the raw query + n := r.FormValue("s") + s := getSeederByName(n) + if s == nil { + writeHeader(w, r) + fmt.Fprintf(w, "No seeder found: %s", html.EscapeString(n)) + writeFooter(w, r, st) + return + } + // FIXME - This is ugly code and needs to be cleaned up a lot - // get v4 std addresses - v4std := getv4stdRR() - v4non := getv4nonRR() - v6std := getv6stdRR() - v6non := getv6nonRR() + config.dnsmtx.RLock() + // if the dns map does not have a key for the request it will return an empty slice + v4std := config.dns[s.dnsHost+".A"] + v4non := config.dns["nonstd."+s.dnsHost+".A"] + v6std := config.dns[s.dnsHost+".AAAA"] + v6non := config.dns["nonstd."+s.dnsHost+".AAAA"] + config.dnsmtx.RUnlock() var v4stdstr, v4nonstr []string var v6stdstr, v6nonstr []string @@ -172,28 +185,39 @@ func statusNGHandler(w http.ResponseWriter, r *http.Request) { } type webstatus struct { - Key string - Value string + Key string + Value string + Seeder string } func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { startT := time.Now() + // read the seeder name + n := r.FormValue("s") + s := getSeederByName(n) + if s == nil { + writeHeader(w, r) + fmt.Fprintf(w, "No seeder found called %s", html.EscapeString(n)) + writeFooter(w, r, startT) + return + } + // gather all the info before writing anything to the remote browser - ws := generateWebStatus(status) + ws := generateWebStatus(s, status) st := `
- + {{range .}}
TwisteeNode Summary
- {{.Key}} + {{.Key}} {{.Value}} @@ -207,18 +231,18 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { writeHeader(w, r) if len(ws) == 0 { - fmt.Fprintf(w, "No Twistees found with this status") + fmt.Fprintf(w, "No Nodes found with this status") } else { switch status { case statusRG: - fmt.Fprintf(w, "
Twistee Status: statusRG - (Reported Good) Have not been able to get addresses yet
") + fmt.Fprintf(w, "
Node Status: statusRG - (Reported Good) Have not been able to get addresses yet
") case statusCG: - fmt.Fprintf(w, "
Twistee Status: statusCG - (Currently Good) Able to connect and get addresses
") + fmt.Fprintf(w, "
Node Status: statusCG - (Currently Good) Able to connect and get addresses
") case statusWG: - fmt.Fprintf(w, "
Twistee Status: statusWG - (Was Good) Was Ok but now can not get addresses
") + fmt.Fprintf(w, "
Node Status: statusWG - (Was Good) Was Ok but now can not get addresses
") case statusNG: - fmt.Fprintf(w, "
Twistee Status: statusNG - (No Good) Unable to get addresses
") + fmt.Fprintf(w, "
Node Status: statusNG - (No Good) Unable to get addresses
") } t := template.New("Status template") t, err := t.Parse(st) @@ -234,7 +258,59 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { writeFooter(w, r, startT) } -// copy Twistee details into a template friendly struct +// generateWebStatus is given a node status and returns a slice of webstatus structures +// ready to be ranged over by an html/template +func generateWebStatus(s *dnsseeder, status uint32) (ws []webstatus) { + + s.mtx.RLock() + defer s.mtx.RUnlock() + + var valueStr string + + for k, v := range s.theList { + if v.status != status { + continue + } + + switch status { + case statusRG: + valueStr = fmt.Sprintf("Fail Count: %v DNS Type: %s", + v.connectFails, + v.dns2str()) + case statusCG: + valueStr = fmt.Sprintf("Remote Version: %v%s Last Block: %v DNS Type: %s", + v.version, + v.strVersion, + v.lastBlock, + v.dns2str()) + + case statusWG: + valueStr = fmt.Sprintf("Last Try: %s ago Last Status: %s\n", + time.Since(v.lastTry).String(), + v.statusStr) + + case statusNG: + valueStr = fmt.Sprintf("Fail Count: %v Last Try: %s ago Last Status: %s\n", + v.connectFails, + time.Since(v.lastTry).String(), + v.statusStr) + + default: + valueStr = "" + } + + ows := webstatus{ + Key: k, + Value: valueStr, + Seeder: s.name, + } + ws = append(ws, ows) + } + + return ws +} + +// copy Node details into a template friendly struct type webtemplate struct { Key string IP string @@ -257,16 +333,16 @@ type webtemplate struct { Nonstdip string } -// reflectHandler processes all requests and returns output in the requested format -func twisteeHandler(w http.ResponseWriter, r *http.Request) { +// nodeHandler displays details about one node +func nodeHandler(w http.ResponseWriter, r *http.Request) { st := time.Now() - twt := ` + ndt := `
- + @@ -285,111 +361,68 @@ func twisteeHandler(w http.ResponseWriter, r *http.Request) {
Twistee {{.Key}}DetailsNode {{.Key}}Details
IP Address{{.IP}}
Port{{.Port}}
` - s := config.seeder + // read the seeder name + n := r.FormValue("s") + s := getSeederByName(n) + if s == nil { + writeHeader(w, r) + fmt.Fprintf(w, "No seeder found called %s", html.EscapeString(n)) + writeFooter(w, r, st) + return + } s.mtx.RLock() defer s.mtx.RUnlock() - // skip the tw= from the raw query - k := html.UnescapeString(r.URL.RawQuery[3:]) + k := r.FormValue("nd") writeHeader(w, r) if _, ok := s.theList[k]; ok == false { - fmt.Fprintf(w, "Sorry there is no Twistee with those details\n") + fmt.Fprintf(w, "Sorry there is no Node with those details\n") } else { - tw := s.theList[k] + nd := s.theList[k] wt := webtemplate{ - IP: tw.na.IP.String(), - Port: tw.na.Port, - Dnstype: tw.dns2str(), - Nonstdip: tw.nonstdIP.String(), - Statusstr: tw.statusStr, - Lastconnect: tw.lastConnect.String(), - Lastconnectago: time.Since(tw.lastConnect).String(), - Lasttry: tw.lastTry.String(), - Lasttryago: time.Since(tw.lastTry).String(), - Crawlstart: tw.crawlStart.String(), - Crawlstartago: time.Since(tw.crawlStart).String(), - Connectfails: tw.connectFails, - Crawlactive: tw.crawlActive, - Version: tw.version, - Strversion: tw.strVersion, - Services: tw.services.String(), - Lastblock: tw.lastBlock, + IP: nd.na.IP.String(), + Port: nd.na.Port, + Dnstype: nd.dns2str(), + Nonstdip: nd.nonstdIP.String(), + Statusstr: nd.statusStr, + Lastconnect: nd.lastConnect.String(), + Lastconnectago: time.Since(nd.lastConnect).String(), + Lasttry: nd.lastTry.String(), + Lasttryago: time.Since(nd.lastTry).String(), + Crawlstart: nd.crawlStart.String(), + Crawlstartago: time.Since(nd.crawlStart).String(), + Connectfails: nd.connectFails, + Crawlactive: nd.crawlActive, + Version: nd.version, + Strversion: nd.strVersion, + Services: nd.services.String(), + Lastblock: nd.lastBlock, } - // display details for the Twistee - t := template.New("Twistee template") - t, err := t.Parse(twt) + // display details for the Node + t := template.New("Node template") + t, err := t.Parse(ndt) if err != nil { - log.Printf("error parsing Twistee template %v\n", err) + log.Printf("error parsing Node template %v\n", err) } err = t.Execute(w, wt) if err != nil { - log.Printf("error executing Twistee template %v\n", err) + log.Printf("error executing Node template %v\n", err) } } writeFooter(w, r, st) } -// generateWebStatus is given a twistee status and returns a slice of webstatus structures -// ready to be ranged over by an html/template -func generateWebStatus(status uint32) (ws []webstatus) { - - s := config.seeder - - s.mtx.RLock() - defer s.mtx.RUnlock() - - var valueStr string - - for k, v := range s.theList { - if v.status != status { - continue - } +// summaryHandler displays details about one node +func summaryHandler(w http.ResponseWriter, r *http.Request) { - switch status { - case statusRG: - valueStr = fmt.Sprintf("Fail Count: %v DNS Type: %s", - v.connectFails, - v.dns2str()) - case statusCG: - valueStr = fmt.Sprintf("Remote Version: %v%s Last Block: %v DNS Type: %s", - v.version, - v.strVersion, - v.lastBlock, - v.dns2str()) - - case statusWG: - valueStr = fmt.Sprintf("Last Try: %s ago Last Status: %s\n", - time.Since(v.lastTry).String(), - v.statusStr) - - case statusNG: - valueStr = fmt.Sprintf("Fail Count: %v Last Try: %s ago Last Status: %s\n", - v.connectFails, - time.Since(v.lastTry).String(), - v.statusStr) - - default: - valueStr = "" - } - - ows := webstatus{ - Key: k, - Value: valueStr, - } - ws = append(ws, ows) - } - - return ws -} - -// genHeader will output the standard header -func writeHeader(w http.ResponseWriter, r *http.Request) { + st := time.Now() var hc struct { + Name string RG uint32 RGS uint32 CG uint32 @@ -406,65 +439,85 @@ func writeHeader(w http.ResponseWriter, r *http.Request) { DNSTotal uint32 } - // fill the structs so they can be displayed via the template - counts.mtx.RLock() - hc.RG = counts.TwStatus[statusRG] - hc.RGS = counts.TwStarts[statusRG] - hc.CG = counts.TwStatus[statusCG] - hc.CGS = counts.TwStarts[statusCG] - hc.WG = counts.TwStatus[statusWG] - hc.WGS = counts.TwStarts[statusWG] - hc.NG = counts.TwStatus[statusNG] - hc.NGS = counts.TwStarts[statusNG] - hc.Total = hc.RG + hc.CG + hc.WG + hc.NG - - hc.V4Std = counts.DNSCounts[dnsV4Std] - hc.V4Non = counts.DNSCounts[dnsV4Non] - hc.V6Std = counts.DNSCounts[dnsV6Std] - hc.V6Non = counts.DNSCounts[dnsV6Non] - hc.DNSTotal = hc.V4Std + hc.V4Non + hc.V6Std + hc.V6Non - counts.mtx.RUnlock() - - // we are using basic and simple html here. No fancy graphics or css - h := ` - - dnsseeder -
- statusRG - statusCG - statusWG - statusNG - DNS -
+ writeHeader(w, r) + // loop through each of the seeders + for _, s := range config.seeders { + + hc.Name = s.name + // fill the structs so they can be displayed via the template + s.counts.mtx.RLock() + hc.RG = s.counts.NdStatus[statusRG] + hc.RGS = s.counts.NdStarts[statusRG] + hc.CG = s.counts.NdStatus[statusCG] + hc.CGS = s.counts.NdStarts[statusCG] + hc.WG = s.counts.NdStatus[statusWG] + hc.WGS = s.counts.NdStarts[statusWG] + hc.NG = s.counts.NdStatus[statusNG] + hc.NGS = s.counts.NdStarts[statusNG] + hc.Total = hc.RG + hc.CG + hc.WG + hc.NG + + hc.V4Std = s.counts.DNSCounts[dnsV4Std] + hc.V4Non = s.counts.DNSCounts[dnsV4Non] + hc.V6Std = s.counts.DNSCounts[dnsV6Std] + hc.V6Non = s.counts.DNSCounts[dnsV6Non] + hc.DNSTotal = hc.V4Std + hc.V4Non + hc.V6Std + hc.V6Non + s.counts.mtx.RUnlock() + + // we are using basic and simple html here. No fancy graphics or css + sp := ` + Stats for seeder: {{.Name}} +
- Twistee Stats (count/started)
+ Node Stats (count/started)
- + + + + +
RG: {{.RG}}/{{.RGS}}CG: {{.CG}}/{{.CGS}}WG: {{.WG}}/{{.WGS}}NG: {{.NG}}/{{.NGS}}Total: {{.Total}}RG: {{.RG}}/{{.RGS}}CG: {{.CG}}/{{.CGS}}WG: {{.WG}}/{{.WGS}}NG: {{.NG}}/{{.NGS}}Total: {{.Total}}
DNS Requests
- + + + + +
V4 Std: {{.V4Std}}V4 Non: {{.V4Non}}V6 Std: {{.V6Std}}V6 Non: {{.V6Non}}Total: {{.DNSTotal}}V4 Std: {{.V4Std}}V4 Non: {{.V4Non}}V6 Std: {{.V6Std}}V6 Non: {{.V6Non}}Total: {{.DNSTotal}}
-
` + t := template.New("Header template") + t, err := t.Parse(sp) + if err != nil { + log.Printf("error parsing summary template %v\n", err) + } - t := template.New("Header template") - t, err := t.Parse(h) - if err != nil { - log.Printf("error parsing template %v\n", err) - } - - err = t.Execute(w, hc) - if err != nil { - log.Printf("error executing template %v\n", err) + err = t.Execute(w, hc) + if err != nil { + log.Printf("error executing summary template %v\n", err) + } } + writeFooter(w, r, st) +} +// writeHeader will output the standard header +func writeHeader(w http.ResponseWriter, r *http.Request) { + // we are using basic and simple html here. No fancy graphics or css + h := ` + + dnsseeder +
+ Summary +
+
+
+` + fmt.Fprintf(w, h) } -// genFooter will output the standard footer +// writeFooter will output the standard footer func writeFooter(w http.ResponseWriter, r *http.Request, st time.Time) { // Footer needs to be exported for template processing to work @@ -483,7 +536,7 @@ func writeFooter(w http.ResponseWriter, r *http.Request, st time.Time) {
` - Footer.Uptime = time.Since(config.seeder.uptime).String() + Footer.Uptime = time.Since(config.uptime).String() Footer.Version = config.version Footer.Rt = time.Since(st).String() diff --git a/main.go b/main.go index e9f21e0..4c4dafb 100644 --- a/main.go +++ b/main.go @@ -9,76 +9,86 @@ import ( "log" "os" "os/signal" + "strings" "sync" + "sync/atomic" "syscall" "time" "github.com/miekg/dns" ) -// twCounts holds various statistics about the running system -type twCounts struct { - TwStatus []uint32 - TwStarts []uint32 - DNSCounts []uint32 - mtx sync.RWMutex +// ndCounts holds various statistics about the running system +type NodeCounts struct { + NdStatus []uint32 // number of nodes at each of the 4 statuses - RG, CG, WG, NG + NdStarts []uint32 // number of crawles started last startcrawlers run + DNSCounts []uint32 // number of dns requests for each dns type - dnsV4Std, dnsV4Non, dnsV6Std, dnsV6Non + mtx sync.RWMutex // protect the structures } // configData holds information on the application type configData struct { - host string - port string - http string - version string - verbose bool - debug bool - stats bool - seeder *dnsseeder + uptime time.Time // application start time + port string // port for the dns server to listen on + http string // port for the web server to listen on + version string // application version + verbose bool // verbose output cmdline option + debug bool // debug cmdline option + stats bool // stats cmdline option + seeders map[string]*dnsseeder // holds a pointer to all the current seeders + smtx sync.RWMutex // protect the seeders map + dns map[string][]dns.RR // holds details of all the currently served dns records + dnsmtx sync.RWMutex // protect the dns map + dnsUnknown uint64 // the number of dns requests for we are not configured to handle } var config configData -var counts twCounts -var nwname string +var netfile string func main() { + var j bool + // FIXME - update with git hash during build config.version = "0.6.0" + config.uptime = time.Now() - // initialize the stats counters - counts.TwStatus = make([]uint32, maxStatusTypes) - counts.TwStarts = make([]uint32, maxStatusTypes) - counts.DNSCounts = make([]uint32, maxDNSTypes) - - flag.StringVar(&nwname, "net", "", "Preconfigured Network config") - flag.StringVar(&config.host, "h", "", "DNS host to serve") + flag.StringVar(&netfile, "netfile", "", "List of json config files to load") flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on") flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running") + flag.BoolVar(&j, "j", false, "Write network template file (dnsseeder.json) and exit") flag.BoolVar(&config.verbose, "v", false, "Display verbose output") flag.BoolVar(&config.debug, "d", false, "Display debug output") flag.BoolVar(&config.stats, "s", false, "Display stats output") flag.Parse() - if config.host == "" { - fmt.Printf("error - no hostname provided\n") - os.Exit(1) + if j == true { + createNetFile() + fmt.Printf("Template file has been created\n") + os.Exit(0) } // configure the network options so we can start crawling - thenet := selectNetwork(nwname) - if thenet == nil { - fmt.Printf("Error - No valid network specified. Please add -net= from one of the following:\n") - for _, n := range getNetworkNames() { - fmt.Printf("%s\n", n) - } + netwFiles := strings.Split(netfile, ",") + if len(netwFiles) == 0 { + fmt.Printf("Error - No filenames specified. Please add -net= to load these files\n") os.Exit(1) } - // init the seeder - config.seeder = &dnsseeder{} - config.seeder.theList = make(map[string]*twistee) - config.seeder.uptime = time.Now() - config.seeder.net = thenet + config.seeders = make(map[string]*dnsseeder) + config.dns = make(map[string][]dns.RR) + + for _, nwFile := range netwFiles { + nnw, err := loadNetwork(nwFile) + if err != nil { + fmt.Printf("Error loading data from netfile %s - %v\n", nwFile, err) + os.Exit(1) + } + if nnw != nil { + // FIXME - lock this + config.seeders[nnw.name] = nnw + } + } if config.debug == true { config.verbose = true @@ -88,37 +98,38 @@ func main() { config.stats = true } - log.Printf("Starting dnsseeder system for host %s.\n", config.host) - if config.verbose == false { log.Printf("status - Running in quiet mode with limited output produced\n") } else { - log.Printf("status - system is configured for %s\n", config.seeder.net.name) + for _, v := range config.seeders { + log.Printf("status - system is configured for network: %s\n", v.name) + } } // start the web interface if we want it running if config.http != "" { go startHTTP(config.http) } + // start dns server - dns.HandleFunc("nonstd."+config.host, handleDNSNon) - dns.HandleFunc(config.host, handleDNSStd) + dns.HandleFunc(".", handleDNS) go serve("udp", config.port) //go serve("tcp", config.port) // seed the seeder with some ip addresses - config.seeder.initCrawlers() - // start first crawl - config.seeder.startCrawlers() + for _, s := range config.seeders { + s.initCrawlers() + s.startCrawlers() + } sig := make(chan os.Signal) signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) - // extract good dns records from all twistees on regular basis + // extract good dns records from all nodes on regular basis dnsChan := time.NewTicker(time.Second * dnsDelay).C // used to start crawlers on a regular basis crawlChan := time.NewTicker(time.Second * crawlDelay).C - // used to remove old statusNG twistees that have reached fail count + // used to remove old statusNG nodes that have reached fail count auditChan := time.NewTicker(time.Minute * auditDelay).C dowhile := true @@ -128,40 +139,88 @@ func main() { dowhile = false case <-auditChan: if config.debug { - log.Printf("debug - Audit twistees timer triggered\n") + log.Printf("debug - Audit nodes timer triggered\n") + } + for _, s := range config.seeders { + // FIXME goroutines for these + s.auditNodes() } - config.seeder.auditClients() case <-dnsChan: if config.debug { log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n") } - config.seeder.loadDNS() + for _, s := range config.seeders { + s.loadDNS() + } case <-crawlChan: if config.debug { log.Printf("debug - Start crawlers timer triggered\n") } - config.seeder.startCrawlers() + for _, s := range config.seeders { + s.startCrawlers() + } } } // FIXME - call dns server.Shutdown() fmt.Printf("\nProgram exiting. Bye\n") } -// updateTwCounts runs in a goroutine and updates the global stats with the lates +// updateNodeCounts runs in a goroutine and updates the global stats with the latest // counts from a startCrawlers run -func updateTwCounts(status, total, started uint32) { +func updateNodeCounts(s *dnsseeder, status, total, started uint32) { // update the stats counters - counts.mtx.Lock() - counts.TwStatus[status] = total - counts.TwStarts[status] = started - counts.mtx.Unlock() + s.counts.mtx.Lock() + s.counts.NdStatus[status] = total + s.counts.NdStarts[status] = started + s.counts.mtx.Unlock() } // updateDNSCounts runs in a goroutine and updates the global stats for the number of DNS requests -func updateDNSCounts(dnsType uint32) { - counts.mtx.Lock() - counts.DNSCounts[dnsType]++ - counts.mtx.Unlock() +func updateDNSCounts(name, qtype string) { + var ndType uint32 + var counted bool + + nonstd := strings.HasPrefix(name, "nonstd.") + + switch qtype { + case "A": + if nonstd { + ndType = dnsV4Non + } else { + ndType = dnsV4Std + } + case "AAAA": + if nonstd { + ndType = dnsV6Non + } else { + ndType = dnsV6Std + } + default: + ndType = dnsInvalid + } + + // for DNS requests we do not have a reference to a seeder so we have to find it + for _, s := range config.seeders { + s.counts.mtx.Lock() + + if name == s.dnsHost || name == "nonstd."+s.dnsHost { + s.counts.DNSCounts[ndType]++ + counted = true + } + s.counts.mtx.Unlock() + } + if counted != true { + atomic.AddUint64(&config.dnsUnknown, 1) + } +} + +func getSeederByName(name string) *dnsseeder { + for _, s := range config.seeders { + if s.name == name { + return s + } + } + return nil } /* diff --git a/network.go b/network.go index fca919d..d480816 100644 --- a/network.go +++ b/network.go @@ -1,73 +1,142 @@ package main import ( + "encoding/json" + "errors" + "fmt" "github.com/btcsuite/btcd/wire" + "log" + "os" + "strconv" ) -// network struct holds config details for the network the seeder is using -type network struct { - id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages - maxSize int // max number of clients before we start restricting new entries - port uint16 // default network port this network uses - pver uint32 // minimum block height for the network - ttl uint32 // DNS TTL to use for this network - name string // Short name for the network - description string // Long description for the network - seeders []string // slice of seeders to pull ip addresses when starting this seeder - maxStart []uint32 // max number of goroutines to start each run for each status type - delay []int64 // number of seconds to wait before we connect to a known client for each status +// JNetwork is the exported struct that is read from the network file +type JNetwork struct { + Name string + Desc string + Id string + Port uint16 + Pver uint32 + DNSName string + TTL uint32 + Seeder1 string + Seeder2 string + Seeder3 string } -// getNetworkNames returns a slice of the networks that have been configured -func getNetworkNames() []string { - return []string{"twister", "bitcoin", "bitcoin-testnet"} +func createNetFile() { + // create a standard json template file that can be loaded into the app + + // create a struct to encode with json + jnw := &JNetwork{ + Id: "0xabcdef01", + Port: 1234, + Pver: 70001, + TTL: 600, + DNSName: "seeder.example.com", + Name: "SeederNet", + Desc: "Description of SeederNet", + Seeder1: "seeder1.example.com", + Seeder2: "seed1.bob.com", + Seeder3: "seed2.example.com", + } + + f, err := os.Create("dnsseeder.json") + if err != nil { + log.Printf("error creating template file: %v\n", err) + } + defer f.Close() + + j, jerr := json.MarshalIndent(jnw, "", " ") + if jerr != nil { + log.Printf("error parsing json: %v\n", err) + } + _, ferr := f.Write(j) + if ferr != nil { + log.Printf("error writing to template file: %v\n", err) + } } -// selectNetwork will return a network struct for a given network -func selectNetwork(name string) *network { - switch name { - case "twister": - return &network{ - id: 0xd2bbdaf0, - port: 28333, - pver: 60000, - ttl: 600, - maxSize: 1000, - name: "TwisterNet", - description: "Twister P2P Net", - seeders: []string{"seed2.twister.net.co", "seed.twister.net.co", "seed3.twister.net.co"}, - maxStart: []uint32{15, 15, 15, 30}, - delay: []int64{184, 678, 237, 1876}, - } - case "bitcoin": - return &network{ - id: 0xd9b4bef9, - port: 8333, - pver: 70001, - ttl: 900, - maxSize: 1250, - name: "BitcoinMainNet", - description: "Bitcoin Main Net", - seeders: []string{"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be"}, - maxStart: []uint32{20, 20, 20, 30}, - delay: []int64{210, 789, 234, 1876}, +func loadNetwork(fName string) (*dnsseeder, error) { + nwFile, err := os.Open(fName) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error reading network file: %v", err)) + } + + defer nwFile.Close() + + var jnw JNetwork + + jsonParser := json.NewDecoder(nwFile) + if err = jsonParser.Decode(&jnw); err != nil { + return nil, errors.New(fmt.Sprintf("Error decoding network file: %v", err)) + } + + return initNetwork(jnw, jnw.Name) +} + +func initNetwork(jnw JNetwork, name string) (*dnsseeder, error) { + + if jnw.Port == 0 { + return nil, errors.New(fmt.Sprintf("Invalid port supplied: %v", jnw.Port)) + } + + if jnw.DNSName == "" { + return nil, errors.New(fmt.Sprintf("No DNS Hostname supplied")) + } + + if _, ok := config.seeders[jnw.Name]; ok { + return nil, errors.New(fmt.Sprintf("Name already exists from previous file - %s", jnw.Name)) + } + + // init the seeder + seeder := &dnsseeder{} + seeder.theList = make(map[string]*node) + seeder.port = jnw.Port + seeder.pver = jnw.Pver + seeder.ttl = jnw.TTL + seeder.name = jnw.Name + seeder.desc = jnw.Desc + seeder.dnsHost = jnw.DNSName + + // conver the network magic number to a Uint32 + t1, err := strconv.ParseUint(jnw.Id, 0, 32) + if err != nil { + return nil, errors.New(fmt.Sprintf("Error converting Network Magic number: %v", err)) + } + seeder.id = wire.BitcoinNet(t1) + + // load the seeder dns + seeder.seeders = make([]string, 3) + seeder.seeders[0] = jnw.Seeder1 + seeder.seeders[1] = jnw.Seeder2 + seeder.seeders[2] = jnw.Seeder3 + + // add some checks to the start & delay values to keep them sane + seeder.maxStart = []uint32{20, 20, 20, 30} + seeder.delay = []int64{210, 789, 234, 1876} + seeder.maxSize = 1250 + + // initialize the stats counters + seeder.counts.NdStatus = make([]uint32, maxStatusTypes) + seeder.counts.NdStarts = make([]uint32, maxStatusTypes) + seeder.counts.DNSCounts = make([]uint32, maxDNSTypes) + + // some sanity checks on the loaded config options + if seeder.ttl < 60 { + seeder.ttl = 60 + } + // check for duplicates + for _, v := range config.seeders { + if v.id == seeder.id { + return nil, errors.New(fmt.Sprintf("Duplicate Magic id. Already loaded for %s so can not be used for %s", v.id, v.name, seeder.name)) } - case "bitcoin-testnet": - return &network{ - id: 0xdab5bffa, - port: 18333, - pver: 70001, - ttl: 300, - maxSize: 250, - name: "BitcoinTestNet", - description: "Bitcoin Test Net", - seeders: []string{"testnet-seed.alexykot.me", "testnet-seed.bitcoin.petertodd.org", "testnet-seed.bluematt.me", "testnet-seed.bitcoin.schildbach.de"}, - maxStart: []uint32{15, 15, 15, 30}, - delay: []int64{184, 678, 237, 1876}, + if v.dnsHost == seeder.dnsHost { + return nil, errors.New(fmt.Sprintf("Duplicate DNS names. Already loaded %s for %s so can not be used for %s", v.dnsHost, v.name, seeder.name)) } - default: - return nil } + + return seeder, nil } /* diff --git a/twistee.go b/node.go similarity index 90% rename from twistee.go rename to node.go index 7b8b425..95fba6c 100644 --- a/twistee.go +++ b/node.go @@ -7,8 +7,8 @@ import ( "github.com/btcsuite/btcd/wire" ) -// Twistee struct contains details on one twister client -type twistee struct { +// Node struct contains details on one client +type node struct { na *wire.NetAddress // holds ip address & port details lastConnect time.Time // last time we sucessfully connected to this client lastTry time.Time // last time we tried to connect to this client @@ -28,8 +28,8 @@ type twistee struct { } // status2str will return the string description of the status -func (tw twistee) status2str() string { - switch tw.status { +func (nd node) status2str() string { + switch nd.status { case statusRG: return "statusRG" case statusCG: @@ -44,8 +44,8 @@ func (tw twistee) status2str() string { } // dns2str will return the string description of the dns type -func (tw twistee) dns2str() string { - switch tw.dnsType { +func (nd node) dns2str() string { + switch nd.dnsType { case dnsV4Std: return "v4 standard port" case dnsV4Non: diff --git a/seeder.go b/seeder.go index 13fd848..a54075f 100644 --- a/seeder.go +++ b/seeder.go @@ -22,9 +22,9 @@ const ( auditDelay = 22 // minutes between audit channel ticks dnsDelay = 57 // seconds between updates to active dns record list - maxFails = 58 // max number of connect fails before we delete a twistee. Just over 24 hours(checked every 33 minutes) + maxFails = 58 // max number of connect fails before we delete a node. Just over 24 hours(checked every 33 minutes) - maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout + maxTo = 250 // max seconds (4min 10 sec) for all comms to node to complete before we timeout ) const ( @@ -37,19 +37,37 @@ const ( ) const ( - // twistee status - statusRG = iota // reported good status. A remote twistee has reported this ip but we have not connected - statusCG // confirmed good. We have connected to the twistee and received addresses - statusWG // was good. Twistee was confirmed good but now having problems + // node status + statusRG = iota // reported good status. A remote node has reported this ip but we have not connected + statusCG // confirmed good. We have connected to the node and received addresses + statusWG // was good. node was confirmed good but now having problems statusNG // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses maxStatusTypes // used in main to allocate slice ) +/* NdCounts holds various statistics about the running system +type NdCounts struct { + NdStatus []uint32 + NdStarts []uint32 + DNSCounts []uint32 + mtx sync.RWMutex +} +*/ type dnsseeder struct { - net *network // network struct with config options for this network - uptime time.Time // as the name says - theList map[string]*twistee // the list of current clients - mtx sync.RWMutex + id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages + theList map[string]*node // the list of current nodes + mtx sync.RWMutex // protect thelist + maxSize int // max number of clients before we start restricting new entries + port uint16 // default network port this seeder uses + pver uint32 // minimum block height for the seeder + ttl uint32 // DNS TTL to use for this seeder + dnsHost string // dns host we will serve results for this domain + name string // Short name for the network + desc string // Long description for the network + seeders []string // slice of seeders to pull ip addresses when starting this seeder + maxStart []uint32 // max number of goroutines to start each run for each status type + delay []int64 // number of seconds to wait before we connect to a known client for each status + counts NodeCounts // structure to hold stats for this seeder } // initCrawlers needs to be run before the startCrawlers so it can get @@ -57,28 +75,35 @@ type dnsseeder struct { // start the crawl process func (s *dnsseeder) initCrawlers() { - // get a list of permenant seeders - seeders := s.net.seeders - - for _, aseeder := range seeders { + for _, aseeder := range s.seeders { c := 0 + if aseeder == "" { + continue + } newRRs, err := net.LookupHost(aseeder) if err != nil { - log.Printf("status - unable to do initial lookup to seeder %s %v\n", aseeder, err) + log.Printf("%s: unable to do initial lookup to seeder %s %v\n", s.name, aseeder, err) continue } for _, ip := range newRRs { if newIP := net.ParseIP(ip); newIP != nil { // 1 at the end is the services flag - if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, s.net.port, 1)); x == true { + if x := s.addNa(wire.NewNetAddressIPPort(newIP, s.port, 1)); x == true { c++ } } } if config.verbose { - log.Printf("status - completed import of %v addresses from %s\n", c, aseeder) + log.Printf("%s: completed import of %v addresses from %s\n", s.name, c, aseeder) + } + } + if len(s.theList) == 0 { + log.Printf("%s: Error: No ip addresses from seeders so I have nothing to crawl.\n", s.name) + for _, v := range s.seeders { + log.Printf("%s: Seeder: %s\n", s.name, v) + } } } @@ -90,7 +115,7 @@ func (s *dnsseeder) startCrawlers() { tcount := len(s.theList) if tcount == 0 { if config.debug { - log.Printf("debug - startCrawlers fail: no twistees available\n") + log.Printf("debug - startCrawlers fail: no node ailable\n") } return } @@ -104,10 +129,10 @@ func (s *dnsseeder) startCrawlers() { started uint32 // count of goroutines started for this type delay int64 // number of second since last try }{ - {"statusRG", statusRG, s.net.maxStart[statusRG], 0, 0, s.net.delay[statusRG]}, - {"statusCG", statusCG, s.net.maxStart[statusCG], 0, 0, s.net.delay[statusCG]}, - {"statusWG", statusWG, s.net.maxStart[statusWG], 0, 0, s.net.delay[statusWG]}, - {"statusNG", statusNG, s.net.maxStart[statusNG], 0, 0, s.net.delay[statusNG]}, + {"statusRG", statusRG, s.maxStart[statusRG], 0, 0, s.delay[statusRG]}, + {"statusCG", statusCG, s.maxStart[statusCG], 0, 0, s.delay[statusCG]}, + {"statusWG", statusWG, s.maxStart[statusWG], 0, 0, s.delay[statusWG]}, + {"statusNG", statusNG, s.maxStart[statusNG], 0, 0, s.delay[statusNG]}, } s.mtx.RLock() @@ -118,16 +143,16 @@ func (s *dnsseeder) startCrawlers() { // range on a map will not return items in the same order each time // so this is a random'ish selection - for _, tw := range s.theList { + for _, nd := range s.theList { - if tw.status != c.status { + if nd.status != c.status { continue } // stats count c.totalCount++ - if tw.crawlActive == true { + if nd.crawlActive == true { continue } @@ -135,23 +160,23 @@ func (s *dnsseeder) startCrawlers() { continue } - if (time.Now().Unix() - c.delay) <= tw.lastTry.Unix() { + if (time.Now().Unix() - c.delay) <= nd.lastTry.Unix() { continue } - // all looks good so start a go routine to crawl the remote twistee - go crawlTwistee(s, tw) + // all looks good so start a go routine to crawl the remote node + go crawlNode(s, nd) c.started++ } - log.Printf("stats - started crawler: %s total: %v started: %v\n", c.desc, c.totalCount, c.started) + log.Printf("%s: started crawler: %s total: %v started: %v\n", s.name, c.desc, c.totalCount, c.started) // update the global stats in another goroutine to free the main goroutine // for other work - go updateTwCounts(c.status, c.totalCount, c.started) + go updateNodeCounts(s, c.status, c.totalCount, c.started) } - log.Printf("stats - crawlers started. total twistees: %d\n", tcount) + log.Printf("%s: crawlers started. total clients: %d\n", s.name, tcount) // returns and read lock released } @@ -191,7 +216,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { return false } - nt := twistee{ + nt := node{ na: nNa, lastConnect: time.Now(), version: 0, @@ -203,7 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { // select the dns type based on the remote address type and port if x := nt.na.IP.To4(); x == nil { // not ipv4 - if nNa.Port != s.net.port { + if nNa.Port != s.port { nt.dnsType = dnsV6Non // produce the nonstdIP @@ -214,7 +239,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { } } else { // ipv4 - if nNa.Port != s.net.port { + if nNa.Port != s.port { nt.dnsType = dnsV4Non // force ipv4 address into a 4 byte buffer @@ -270,7 +295,7 @@ func crc16(bs []byte) uint16 { return crc } -func (s *dnsseeder) auditClients() { +func (s *dnsseeder) auditNodes() { c := 0 @@ -280,64 +305,64 @@ func (s *dnsseeder) auditClients() { // cgGoal is 75% of the max statusCG clients we can crawl with the current network delay & maxStart settings. // This allows us to cycle statusCG users to keep the list fresh - cgGoal := int(float64(float64(s.net.delay[statusCG]/crawlDelay)*float64(s.net.maxStart[statusCG])) * 0.75) + cgGoal := int(float64(float64(s.delay[statusCG]/crawlDelay)*float64(s.maxStart[statusCG])) * 0.75) cgCount := 0 - log.Printf("status - Audit start. statusCG Goal: %v System Uptime: %s\n", cgGoal, time.Since(s.uptime).String()) + log.Printf("%s: Audit start. statusCG Goal: %v System Uptime: %s\n", s.name, cgGoal, time.Since(config.uptime).String()) s.mtx.Lock() defer s.mtx.Unlock() - for k, tw := range s.theList { + for k, nd := range s.theList { - if tw.crawlActive == true { - if time.Now().Unix()-tw.crawlStart.Unix() >= 300 { + if nd.crawlActive == true { + if time.Now().Unix()-nd.crawlStart.Unix() >= 300 { log.Printf("warning - long running crawl > 5 minutes ====\n- %s status:rating:fails %v:%v:%v crawl start: %s last status: %s\n====\n", k, - tw.status, - tw.rating, - tw.connectFails, - tw.crawlStart.String(), - tw.statusStr) + nd.status, + nd.rating, + nd.connectFails, + nd.crawlStart.String(), + nd.statusStr) } } - // Audit task is to remove clients that we have not been able to connect to - if tw.status == statusNG && tw.connectFails > maxFails { + // Audit task is to remove node that we have not been able to connect to + if nd.status == statusNG && nd.connectFails > maxFails { if config.verbose { - log.Printf("status - purging twistee %s after %v failed connections\n", k, tw.connectFails) + log.Printf("%s: purging node %s after %v failed connections\n", s.name, k, nd.connectFails) } c++ - // remove the map entry and mark the old twistee as + // remove the map entry and mark the old node as // nil so garbage collector will remove it s.theList[k] = nil delete(s.theList, k) } // If seeder is full then remove old NG clients and fill up with possible new CG clients - if tw.status == statusNG && iAmFull { + if nd.status == statusNG && iAmFull { if config.verbose { - log.Printf("status - seeder full purging twistee %s\n", k) + log.Printf("%s: seeder full purging node %s\n", s.name, k) } c++ - // remove the map entry and mark the old twistee as + // remove the map entry and mark the old node as // nil so garbage collector will remove it s.theList[k] = nil delete(s.theList, k) } // check if we need to purge statusCG to freshen the list - if tw.status == statusCG { + if nd.status == statusCG { if cgCount++; cgCount > cgGoal { // we have enough statusCG clients so purge remaining to cycle through the list if config.verbose { - log.Printf("status - seeder cycle statusCG - purging client %s\n", k) + log.Printf("%s: seeder cycle statusCG - purging node %s\n", s.name, k) } c++ - // remove the map entry and mark the old twistee as + // remove the map entry and mark the old node as // nil so garbage collector will remove it s.theList[k] = nil delete(s.theList, k) @@ -347,7 +372,7 @@ func (s *dnsseeder) auditClients() { } if config.verbose { - log.Printf("status - Audit complete. %v twistees purged\n", c) + log.Printf("%s: Audit complete. %v nodes purged\n", s.name, c) } } @@ -359,7 +384,7 @@ func (s *dnsseeder) loadDNS() { // isFull returns true if the number of remote clients is more than we want to store func (s *dnsseeder) isFull() bool { - if len(s.theList) > s.net.maxSize { + if len(s.theList) > s.maxSize { return true } return false