From 8ce82dda73f41214ee6f81b814bddbd83f41bdf7 Mon Sep 17 00:00:00 2001 From: Lyndsay Roger Date: Wed, 2 Sep 2015 15:09:15 +1200 Subject: [PATCH] Initial support for seeding multiple networks and improve scaling for larger networks --- README.md | 19 ++++--- crawler.go | 45 ++++++++++------- dns.go | 20 ++++---- doc.go | 13 ++++- http.go | 2 - main.go | 45 +++++++++++------ network.go | 75 +++++++++++++++++++++++++++ seeder.go | 145 +++++++++++++++++++++++++++++++++++------------------ twistee.go | 24 ++++----- 9 files changed, 270 insertions(+), 118 deletions(-) create mode 100644 network.go diff --git a/README.md b/README.md index 89bc55d..f8ef26c 100644 --- a/README.md +++ b/README.md @@ -1,14 +1,12 @@ # dnsseeder -Go Language dns seeder for the Twister P2P network - -This is a dns seeder for the [Twister P2P network](http://twister.net.co/) +Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/) It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder -Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/) +This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option. +Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/) -> **NOTE:** This repository is under ongoing development. Stable releases have been tagged and should be used for production systems. ## Installing @@ -36,7 +34,7 @@ The binary will then be available in ${HOME}/go/bin ## Usage - $ dnsseeder -h + $ dnsseeder -h -net An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running. @@ -48,7 +46,8 @@ If you want to be able to view the web interface then add -w port for the web se Command line Options: -h hostname to serve --p port to listen on +-net The network to seed. Currently twister, bitcoin, bitcoin-test +-p port to listen on for DNS requests -d Produce debug output -v Produce verbose output -w Port to listen on for Web Interface @@ -69,16 +68,16 @@ gzip ${LOGDIR}/*.log # pass through the logging level needed if [ -z ${1} ]; then - LOGLV="-v" + THENET="twister" else - LOGLV="${1}" + THENET="${1}" fi cd echo echo "======= Run the Go Language dnsseed =======" echo -${HOME}/go/bin/dnsseeder -h -p ${LOGLV} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log +${HOME}/go/bin/dnsseeder -h -p ${THENET -v} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log ``` diff --git a/crawler.go b/crawler.go index ac37c3e..cab44f3 100644 --- a/crawler.go +++ b/crawler.go @@ -22,7 +22,7 @@ func (e *crawlError) Error() string { // crawlTwistee runs in a goroutine, crawls the remote ip and updates the master // list of currently active addresses -func crawlTwistee(tw *twistee) { +func crawlTwistee(s *dnsseeder, tw *twistee) { tw.crawlActive = true tw.crawlStart = time.Now() @@ -39,7 +39,7 @@ func crawlTwistee(tw *twistee) { } // connect to the remote ip and ask them for their addr list - ras, e := crawlIP(tw) + rna, e := crawlIP(s.net.pver, s.net.id, tw) if e != nil { // update the fact that we have not connected to this twistee @@ -50,9 +50,15 @@ func crawlTwistee(tw *twistee) { // update the status of this failed twistee switch tw.status { case statusRG: - if tw.rating += 25; tw.rating > 30 { - tw.status = statusWG + // if we are full then any RG failures will skip directly to NG + if s.isFull() { + tw.status = statusNG // not able to connect to this twistee so ignore tw.statusTime = time.Now() + } else { + if tw.rating += 25; tw.rating > 30 { + tw.status = statusWG + tw.statusTime = time.Now() + } } case statusCG: if tw.rating += 25; tw.rating >= 50 { @@ -60,7 +66,7 @@ func crawlTwistee(tw *twistee) { tw.statusTime = time.Now() } case statusWG: - if tw.rating += 30; tw.rating >= 100 { + if tw.rating += 15; tw.rating >= 100 { tw.status = statusNG // not able to connect to this twistee so ignore tw.statusTime = time.Now() } @@ -93,11 +99,14 @@ func crawlTwistee(tw *twistee) { added := 0 - // loop through all the received network addresses and add to thelist if not present - for _, na := range ras { - // a new network address so add to the system - if x := config.seeder.addNa(na); x == true { - added++ + // if we are full then skip adding more possible clients + if s.isFull() == false { + // loop through all the received network addresses and add to thelist if not present + for _, na := range rna { + // a new network address so add to the system + if x := s.addNa(na); x == true { + added++ + } } } @@ -108,7 +117,7 @@ func crawlTwistee(tw *twistee) { tw.status, tw.rating, tw.connectFails, - len(ras), + len(rna), added, time.Since(tw.crawlStart).String(), time.Since(cs).String()) @@ -125,7 +134,7 @@ func crawlEnd(tw *twistee) { } // crawlIP retrievs a slice of ip addresses from a client -func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) { +func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) { ip := tw.na.IP.String() port := strconv.Itoa(int(tw.na.Port)) @@ -155,14 +164,14 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) { return nil, &crawlError{"Create NewMsgVersionFromConn", err} } - err = wire.WriteMessage(conn, msgver, pver, twistNet) + err = wire.WriteMessage(conn, msgver, pver, netID) if err != nil { // Log and handle the error return nil, &crawlError{"Write Version Message", err} } // first message received should be version - msg, _, err := wire.ReadMessage(conn, pver, twistNet) + msg, _, err := wire.ReadMessage(conn, pver, netID) if err != nil { // Log and handle the error return nil, &crawlError{"Read message after sending Version", err} @@ -190,13 +199,13 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) { // send verack command msgverack := wire.NewMsgVerAck() - err = wire.WriteMessage(conn, msgverack, pver, twistNet) + err = wire.WriteMessage(conn, msgverack, pver, netID) if err != nil { return nil, &crawlError{"writing message VerAck", err} } // second message received should be verack - msg, _, err = wire.ReadMessage(conn, pver, twistNet) + msg, _, err = wire.ReadMessage(conn, pver, netID) if err != nil { return nil, &crawlError{"reading expected Ver Ack from remote client", err} } @@ -213,7 +222,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) { // send getaddr command msgGetAddr := wire.NewMsgGetAddr() - err = wire.WriteMessage(conn, msgGetAddr, pver, twistNet) + err = wire.WriteMessage(conn, msgGetAddr, pver, netID) if err != nil { return nil, &crawlError{"writing Addr message to remote client", err} } @@ -225,7 +234,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) { // Using the Bitcoin lib for the Twister Net means it does not understand some // of the commands and will error. We can ignore these as we are only // interested in the addr message and its content. - msgaddr, _, _ := wire.ReadMessage(conn, pver, twistNet) + msgaddr, _, _ := wire.ReadMessage(conn, pver, netID) if msgaddr != nil { switch msg := msgaddr.(type) { case *wire.MsgAddr: diff --git a/dns.go b/dns.go index 720f903..11612bb 100644 --- a/dns.go +++ b/dns.go @@ -51,7 +51,7 @@ func updateDNS(s *dnsseeder) { if t == dnsV4Std || t == dnsV4Non { if t == dnsV4Std && tw.dnsType == dnsV4Std { r := new(dns.A) - r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} r.A = tw.na.IP rr4std = append(rr4std, r) numRR++ @@ -60,12 +60,12 @@ func updateDNS(s *dnsseeder) { // if the twistee is using a non standard port then add the encoded port info to DNS if t == dnsV4Non && tw.dnsType == dnsV4Non { r := new(dns.A) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} r.A = tw.na.IP rr4non = append(rr4non, r) numRR++ r = new(dns.A) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl} r.A = tw.nonstdIP rr4non = append(rr4non, r) numRR++ @@ -74,7 +74,7 @@ func updateDNS(s *dnsseeder) { if t == dnsV6Std || t == dnsV6Non { if t == dnsV6Std && tw.dnsType == dnsV6Std { r := new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} r.AAAA = tw.na.IP rr6std = append(rr6std, r) numRR++ @@ -82,12 +82,12 @@ func updateDNS(s *dnsseeder) { // if the twistee is using a non standard port then add the encoded port info to DNS if t == dnsV6Non && tw.dnsType == dnsV6Non { r := new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} r.AAAA = tw.na.IP rr6non = append(rr6non, r) numRR++ r = new(dns.AAAA) - r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60} + r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl} r.AAAA = tw.nonstdIP rr6non = append(rr6non, r) numRR++ @@ -156,8 +156,8 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) { w.WriteMsg(m) - if config.verbose { - log.Printf("status - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) + if config.debug { + log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) } } @@ -195,8 +195,8 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) { w.WriteMsg(m) - if config.verbose { - log.Printf("status - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) + if config.debug { + log.Printf("debug - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) } } diff --git a/doc.go b/doc.go index 241de5e..2d7c5cf 100644 --- a/doc.go +++ b/doc.go @@ -1,8 +1,17 @@ /* -This application provides a seeder service to the Twister Network. +This application provides a DNS seeder service to network based on Bitcoin technology. +For example - http://twister.net.co/ +https://bitcoin.org/ -It crawls the Twister Network for active clients and records their ip address and port. It then replies to DNS queries with the ip addresses. + +This application crawls the Network for active clients and records their ip address and port. It then replies to DNS queries with this information. + +Features: +- Preconfigured support for Twister & Bitcoin networks. use -net to load config data. +- supports ipv4 & ipv6 addresses +- revisits clients on a configurable time basis to make sure they are still available +- Low memory & cpu requirements */ package main diff --git a/http.go b/http.go index 286b5c9..2cc4939 100644 --- a/http.go +++ b/http.go @@ -231,9 +231,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) { } } - writeFooter(w, r, startT) - } // copy Twistee details into a template friendly struct diff --git a/main.go b/main.go index 5df152d..e9f21e0 100644 --- a/main.go +++ b/main.go @@ -38,29 +38,48 @@ type configData struct { var config configData var counts twCounts +var nwname string func main() { // FIXME - update with git hash during build - config.version = "0.5.0" + config.version = "0.6.0" // initialize the stats counters counts.TwStatus = make([]uint32, maxStatusTypes) counts.TwStarts = make([]uint32, maxStatusTypes) counts.DNSCounts = make([]uint32, maxDNSTypes) + flag.StringVar(&nwname, "net", "", "Preconfigured Network config") flag.StringVar(&config.host, "h", "", "DNS host to serve") - flag.StringVar(&config.port, "p", "8053", "Port to listen on") + flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on") + flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running") flag.BoolVar(&config.verbose, "v", false, "Display verbose output") flag.BoolVar(&config.debug, "d", false, "Display debug output") flag.BoolVar(&config.stats, "s", false, "Display stats output") - flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running") flag.Parse() if config.host == "" { - log.Fatalf("error - no hostname provided\n") + fmt.Printf("error - no hostname provided\n") + os.Exit(1) + } + + // configure the network options so we can start crawling + thenet := selectNetwork(nwname) + if thenet == nil { + fmt.Printf("Error - No valid network specified. Please add -net= from one of the following:\n") + for _, n := range getNetworkNames() { + fmt.Printf("%s\n", n) + } + os.Exit(1) } + // init the seeder + config.seeder = &dnsseeder{} + config.seeder.theList = make(map[string]*twistee) + config.seeder.uptime = time.Now() + config.seeder.net = thenet + if config.debug == true { config.verbose = true config.stats = true @@ -73,18 +92,14 @@ func main() { if config.verbose == false { log.Printf("status - Running in quiet mode with limited output produced\n") + } else { + log.Printf("status - system is configured for %s\n", config.seeder.net.name) } // start the web interface if we want it running if config.http != "" { go startHTTP(config.http) } - - // init the seeder - config.seeder = &dnsseeder{} - config.seeder.theList = make(map[string]*twistee) - config.seeder.uptime = time.Now() - // start dns server dns.HandleFunc("nonstd."+config.host, handleDNSNon) dns.HandleFunc(config.host, handleDNSStd) @@ -92,7 +107,7 @@ func main() { //go serve("tcp", config.port) // seed the seeder with some ip addresses - initCrawlers() + config.seeder.initCrawlers() // start first crawl config.seeder.startCrawlers() @@ -100,11 +115,11 @@ func main() { signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM) // extract good dns records from all twistees on regular basis - dnsChan := time.NewTicker(time.Second * 57).C + dnsChan := time.NewTicker(time.Second * dnsDelay).C // used to start crawlers on a regular basis - crawlChan := time.NewTicker(time.Second * 22).C + crawlChan := time.NewTicker(time.Second * crawlDelay).C // used to remove old statusNG twistees that have reached fail count - auditChan := time.NewTicker(time.Hour * 1).C + auditChan := time.NewTicker(time.Minute * auditDelay).C dowhile := true for dowhile == true { @@ -115,7 +130,7 @@ func main() { if config.debug { log.Printf("debug - Audit twistees timer triggered\n") } - config.seeder.auditTwistees() + config.seeder.auditClients() case <-dnsChan: if config.debug { log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n") diff --git a/network.go b/network.go new file mode 100644 index 0000000..fca919d --- /dev/null +++ b/network.go @@ -0,0 +1,75 @@ +package main + +import ( + "github.com/btcsuite/btcd/wire" +) + +// network struct holds config details for the network the seeder is using +type network struct { + id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages + maxSize int // max number of clients before we start restricting new entries + port uint16 // default network port this network uses + pver uint32 // minimum block height for the network + ttl uint32 // DNS TTL to use for this network + name string // Short name for the network + description string // Long description for the network + seeders []string // slice of seeders to pull ip addresses when starting this seeder + maxStart []uint32 // max number of goroutines to start each run for each status type + delay []int64 // number of seconds to wait before we connect to a known client for each status +} + +// getNetworkNames returns a slice of the networks that have been configured +func getNetworkNames() []string { + return []string{"twister", "bitcoin", "bitcoin-testnet"} +} + +// selectNetwork will return a network struct for a given network +func selectNetwork(name string) *network { + switch name { + case "twister": + return &network{ + id: 0xd2bbdaf0, + port: 28333, + pver: 60000, + ttl: 600, + maxSize: 1000, + name: "TwisterNet", + description: "Twister P2P Net", + seeders: []string{"seed2.twister.net.co", "seed.twister.net.co", "seed3.twister.net.co"}, + maxStart: []uint32{15, 15, 15, 30}, + delay: []int64{184, 678, 237, 1876}, + } + case "bitcoin": + return &network{ + id: 0xd9b4bef9, + port: 8333, + pver: 70001, + ttl: 900, + maxSize: 1250, + name: "BitcoinMainNet", + description: "Bitcoin Main Net", + seeders: []string{"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be"}, + maxStart: []uint32{20, 20, 20, 30}, + delay: []int64{210, 789, 234, 1876}, + } + case "bitcoin-testnet": + return &network{ + id: 0xdab5bffa, + port: 18333, + pver: 70001, + ttl: 300, + maxSize: 250, + name: "BitcoinTestNet", + description: "Bitcoin Test Net", + seeders: []string{"testnet-seed.alexykot.me", "testnet-seed.bitcoin.petertodd.org", "testnet-seed.bluematt.me", "testnet-seed.bitcoin.schildbach.de"}, + maxStart: []uint32{15, 15, 15, 30}, + delay: []int64{184, 678, 237, 1876}, + } + default: + return nil + } +} + +/* + + */ diff --git a/seeder.go b/seeder.go index 5a3dca5..13fd848 100644 --- a/seeder.go +++ b/seeder.go @@ -12,68 +12,73 @@ import ( const ( - // TWISTNET Magic number to make it incompatible with the Bitcoin network - twistNet = 0xd2bbdaf0 // NOUNCE is used to check if we connect to ourselves // as we don't listen we can use a fixed value nounce = 0x0539a019ca550825 - pver = 60000 minPort = 0 maxPort = 65535 - twStdPort = 28333 // standard port twister listens on + crawlDelay = 22 // seconds between start crawlwer ticks + auditDelay = 22 // minutes between audit channel ticks + dnsDelay = 57 // seconds between updates to active dns record list maxFails = 58 // max number of connect fails before we delete a twistee. Just over 24 hours(checked every 33 minutes) maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout +) - dnsInvalid = 0 - dnsV4Std = 1 - dnsV4Non = 2 - dnsV6Std = 3 - dnsV6Non = 4 - maxDNSTypes = 5 +const ( + dnsInvalid = iota // + dnsV4Std // ip v4 using network standard port + dnsV4Non // ip v4 using network non standard port + dnsV6Std // ipv6 using network standard port + dnsV6Non // ipv6 using network non standard port + maxDNSTypes // used in main to allocate slice +) +const ( // twistee status - statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected - statusCG = 2 // confirmed good. We have connected to the twistee and received addresses - statusWG = 3 // was good. Twistee was confirmed good but now having problems - statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses - maxStatusTypes = 5 + statusRG = iota // reported good status. A remote twistee has reported this ip but we have not connected + statusCG // confirmed good. We have connected to the twistee and received addresses + statusWG // was good. Twistee was confirmed good but now having problems + statusNG // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses + maxStatusTypes // used in main to allocate slice ) type dnsseeder struct { - uptime time.Time - theList map[string]*twistee + net *network // network struct with config options for this network + uptime time.Time // as the name says + theList map[string]*twistee // the list of current clients mtx sync.RWMutex } // initCrawlers needs to be run before the startCrawlers so it can get // a list of current ip addresses from the other seeders and therefore // start the crawl process -func initCrawlers() { +func (s *dnsseeder) initCrawlers() { - seeders := []string{"seed2.twister.net.co", "seed3.twister.net.co", "seed.twister.net.co"} + // get a list of permenant seeders + seeders := s.net.seeders - for _, seeder := range seeders { + for _, aseeder := range seeders { c := 0 - newRRs, err := net.LookupHost(seeder) + newRRs, err := net.LookupHost(aseeder) if err != nil { - log.Printf("status - unable to do initial lookup to seeder %s %v\n", seeder, err) + log.Printf("status - unable to do initial lookup to seeder %s %v\n", aseeder, err) continue } for _, ip := range newRRs { if newIP := net.ParseIP(ip); newIP != nil { // 1 at the end is the services flag - if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, 28333, 1)); x == true { + if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, s.net.port, 1)); x == true { c++ } } } if config.verbose { - log.Printf("status - completed import of %v addresses from %s\n", c, seeder) + log.Printf("status - completed import of %v addresses from %s\n", c, aseeder) } } } @@ -99,19 +104,20 @@ func (s *dnsseeder) startCrawlers() { started uint32 // count of goroutines started for this type delay int64 // number of second since last try }{ - {"statusRG", statusRG, 10, 0, 0, 184}, - {"statusCG", statusCG, 10, 0, 0, 325}, - {"statusWG", statusWG, 10, 0, 0, 237}, - {"statusNG", statusNG, 20, 0, 0, 1876}, + {"statusRG", statusRG, s.net.maxStart[statusRG], 0, 0, s.net.delay[statusRG]}, + {"statusCG", statusCG, s.net.maxStart[statusCG], 0, 0, s.net.delay[statusCG]}, + {"statusWG", statusWG, s.net.maxStart[statusWG], 0, 0, s.net.delay[statusWG]}, + {"statusNG", statusNG, s.net.maxStart[statusNG], 0, 0, s.net.delay[statusNG]}, } s.mtx.RLock() defer s.mtx.RUnlock() + // step through each of the status types RG, CG, WG, NG for _, c := range crawlers { // range on a map will not return items in the same order each time - // not the best method to randomly pick twistees to crawl. FIXME + // so this is a random'ish selection for _, tw := range s.theList { if tw.status != c.status { @@ -134,7 +140,7 @@ func (s *dnsseeder) startCrawlers() { } // all looks good so start a go routine to crawl the remote twistee - go crawlTwistee(tw) + go crawlTwistee(s, tw) c.started++ } @@ -166,6 +172,12 @@ func (s *dnsseeder) isNaDup(na *wire.NetAddress) bool { // addNa validates and adds a network address to theList func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { + // as this is run in many different goroutines then they may all try and + // add new addresses so do a final check + if s.isFull() { + return false + } + if dup := s.isNaDup(nNa); dup == true { return false } @@ -191,7 +203,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { // select the dns type based on the remote address type and port if x := nt.na.IP.To4(); x == nil { // not ipv4 - if nNa.Port != twStdPort { + if nNa.Port != s.net.port { nt.dnsType = dnsV6Non // produce the nonstdIP @@ -202,7 +214,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { } } else { // ipv4 - if nNa.Port != twStdPort { + if nNa.Port != s.net.port { nt.dnsType = dnsV4Non // force ipv4 address into a 4 byte buffer @@ -216,8 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool { // generate the key and add to theList k := net.JoinHostPort(nNa.IP.String(), strconv.Itoa(int(nNa.Port))) s.mtx.Lock() - // final check to make sure another twistee & goroutine has not already added this twistee - // FIXME migrate to use channels + // final check to make sure another crawl & goroutine has not already added this client if _, dup := s.theList[k]; dup == false { s.theList[k] = &nt } @@ -259,10 +270,20 @@ func crc16(bs []byte) uint16 { return crc } -func (s *dnsseeder) auditTwistees() { +func (s *dnsseeder) auditClients() { c := 0 - log.Printf("status - Audit start. System Uptime: %s\n", time.Since(s.uptime).String()) + + // set this early so for this audit run all NG clients will be purged + // and space will be made for new, possible CG clients + iAmFull := s.isFull() + + // cgGoal is 75% of the max statusCG clients we can crawl with the current network delay & maxStart settings. + // This allows us to cycle statusCG users to keep the list fresh + cgGoal := int(float64(float64(s.net.delay[statusCG]/crawlDelay)*float64(s.net.maxStart[statusCG])) * 0.75) + cgCount := 0 + + log.Printf("status - Audit start. statusCG Goal: %v System Uptime: %s\n", cgGoal, time.Since(s.uptime).String()) s.mtx.Lock() defer s.mtx.Unlock() @@ -280,19 +301,8 @@ func (s *dnsseeder) auditTwistees() { tw.statusStr) } } - if tw.status == statusRG || tw.status == statusWG { - if time.Now().Unix()-tw.statusTime.Unix() >= 900 { - log.Printf("warning - unchanged status > 15 minutes ====\n- %s status:rating:fails %v:%v:%v last status change: %s last status: %s\n====\n", - k, - tw.status, - tw.rating, - tw.connectFails, - tw.statusTime.String(), - tw.statusStr) - } - } - // last audit task is to remove twistees that we can not connect to + // Audit task is to remove clients that we have not been able to connect to if tw.status == statusNG && tw.connectFails > maxFails { if config.verbose { log.Printf("status - purging twistee %s after %v failed connections\n", k, tw.connectFails) @@ -305,6 +315,36 @@ func (s *dnsseeder) auditTwistees() { delete(s.theList, k) } + // If seeder is full then remove old NG clients and fill up with possible new CG clients + if tw.status == statusNG && iAmFull { + if config.verbose { + log.Printf("status - seeder full purging twistee %s\n", k) + } + + c++ + // remove the map entry and mark the old twistee as + // nil so garbage collector will remove it + s.theList[k] = nil + delete(s.theList, k) + } + + // check if we need to purge statusCG to freshen the list + if tw.status == statusCG { + if cgCount++; cgCount > cgGoal { + // we have enough statusCG clients so purge remaining to cycle through the list + if config.verbose { + log.Printf("status - seeder cycle statusCG - purging client %s\n", k) + } + + c++ + // remove the map entry and mark the old twistee as + // nil so garbage collector will remove it + s.theList[k] = nil + delete(s.theList, k) + } + + } + } if config.verbose { log.Printf("status - Audit complete. %v twistees purged\n", c) @@ -314,10 +354,17 @@ func (s *dnsseeder) auditTwistees() { // teatload loads the dns records with time based test data func (s *dnsseeder) loadDNS() { - updateDNS(s) } +// isFull returns true if the number of remote clients is more than we want to store +func (s *dnsseeder) isFull() bool { + if len(s.theList) > s.net.maxSize { + return true + } + return false +} + /* */ diff --git a/twistee.go b/twistee.go index 1b21062..7b8b425 100644 --- a/twistee.go +++ b/twistee.go @@ -9,22 +9,22 @@ import ( // Twistee struct contains details on one twister client type twistee struct { - na *wire.NetAddress - lastConnect time.Time - lastTry time.Time - crawlStart time.Time - statusTime time.Time - crawlActive bool - connectFails uint32 + na *wire.NetAddress // holds ip address & port details + lastConnect time.Time // last time we sucessfully connected to this client + lastTry time.Time // last time we tried to connect to this client + crawlStart time.Time // time when we started the last crawl + statusTime time.Time // time the status was last updated + crawlActive bool // are we currently crawling this client + connectFails uint32 // number of times we have failed to connect to this client statusStr string // string with last error or OK details version int32 // remote client protocol version strVersion string // remote client user agent services wire.ServiceFlag // remote client supported services - lastBlock int32 - status uint32 // rg,cg,wg,ng - rating uint32 // if it reaches 100 then we ban them - nonstdIP net.IP - dnsType uint32 + lastBlock int32 // remote client last block + status uint32 // rg,cg,wg,ng + rating uint32 // if it reaches 100 then we mark them statusNG + nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port + dnsType uint32 // what dns type this client is } // status2str will return the string description of the status