Initial support for seeding multiple networks and improve scaling for larger networks

README.md

@@ -1,15 +1,13 @@
 # dnsseeder
-Go Language dns seeder for the Twister P2P network
+Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/)
-
-This is a dns seeder for the [Twister P2P network](http://twister.net.co/)
 
 It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder
 
+This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option.
+
 Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/)
 
 
-> **NOTE:** This repository is under ongoing development. Stable releases have been tagged and should be used for production systems.
-
 
 ## Installing
 
@@ -36,7 +34,7 @@ The binary will then be available in ${HOME}/go/bin
 
 ## Usage
 
-    $ dnsseeder -h <domain respond to>
+    $ dnsseeder -h <domain respond to> -net <network to seed>
 
 An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running.
 
@@ -48,7 +46,8 @@ If you want to be able to view the web interface then add -w port for the web se
 
 Command line Options:
 -h hostname to serve 
--p port to listen on
+-net The network to seed. Currently twister, bitcoin, bitcoin-test
+-p port to listen on for DNS requests
 -d Produce debug output
 -v Produce verbose output
 -w Port to listen on for Web Interface
@@ -69,16 +68,16 @@ gzip ${LOGDIR}/*.log
 
 # pass through the logging level needed
 if [ -z ${1} ]; then
-        LOGLV="-v"
+        THENET="twister"
 else
-        LOGLV="${1}"
+        THENET="${1}"
 fi
 
 cd
 echo
 echo "======= Run the Go Language dnsseed ======="
 echo
-${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <port.to.listen.on> ${LOGLV} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
+${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <dns.port.to.listen.on> ${THENET -v} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
 
 
 ```

crawler.go

@@ -22,7 +22,7 @@ func (e *crawlError) Error() string {
 
 // crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
 // list of currently active addresses
-func crawlTwistee(tw *twistee) {
+func crawlTwistee(s *dnsseeder, tw *twistee) {
 
 	tw.crawlActive = true
 	tw.crawlStart = time.Now()
@@ -39,7 +39,7 @@ func crawlTwistee(tw *twistee) {
 	}
 
 	// connect to the remote ip and ask them for their addr list
-	ras, e := crawlIP(tw)
+	rna, e := crawlIP(s.net.pver, s.net.id, tw)
 
 	if e != nil {
 		// update the fact that we have not connected to this twistee
@@ -50,9 +50,15 @@ func crawlTwistee(tw *twistee) {
 		// update the status of this failed twistee
 		switch tw.status {
 		case statusRG:
-			if tw.rating += 25; tw.rating > 30 {
+			// if we are full then any RG failures will skip directly to NG
-				tw.status = statusWG
+			if s.isFull() {
+				tw.status = statusNG // not able to connect to this twistee so ignore
 				tw.statusTime = time.Now()
+			} else {
+				if tw.rating += 25; tw.rating > 30 {
+					tw.status = statusWG
+					tw.statusTime = time.Now()
+				}
 			}
 		case statusCG:
 			if tw.rating += 25; tw.rating >= 50 {
@@ -60,7 +66,7 @@ func crawlTwistee(tw *twistee) {
 				tw.statusTime = time.Now()
 			}
 		case statusWG:
-			if tw.rating += 30; tw.rating >= 100 {
+			if tw.rating += 15; tw.rating >= 100 {
 				tw.status = statusNG // not able to connect to this twistee so ignore
 				tw.statusTime = time.Now()
 			}
@@ -93,11 +99,14 @@ func crawlTwistee(tw *twistee) {
 
 	added := 0
 
-	// loop through all the received network addresses and add to thelist if not present
+	// if we are full then skip adding more possible clients
-	for _, na := range ras {
+	if s.isFull() == false {
-		// a new network address so add to the system
+		// loop through all the received network addresses and add to thelist if not present
-		if x := config.seeder.addNa(na); x == true {
+		for _, na := range rna {
-			added++
+			// a new network address so add to the system
+			if x := s.addNa(na); x == true {
+				added++
+			}
 		}
 	}
 
@@ -108,7 +117,7 @@ func crawlTwistee(tw *twistee) {
 			tw.status,
 			tw.rating,
 			tw.connectFails,
-			len(ras),
+			len(rna),
 			added,
 			time.Since(tw.crawlStart).String(),
 			time.Since(cs).String())
@@ -125,7 +134,7 @@ func crawlEnd(tw *twistee) {
 }
 
 // crawlIP retrievs a slice of ip addresses from a client
-func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
+func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) {
 
 	ip := tw.na.IP.String()
 	port := strconv.Itoa(int(tw.na.Port))
@@ -155,14 +164,14 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 		return nil, &crawlError{"Create NewMsgVersionFromConn", err}
 	}
 
-	err = wire.WriteMessage(conn, msgver, pver, twistNet)
+	err = wire.WriteMessage(conn, msgver, pver, netID)
 	if err != nil {
 		// Log and handle the error
 		return nil, &crawlError{"Write Version Message", err}
 	}
 
 	// first message received should be version
-	msg, _, err := wire.ReadMessage(conn, pver, twistNet)
+	msg, _, err := wire.ReadMessage(conn, pver, netID)
 	if err != nil {
 		// Log and handle the error
 		return nil, &crawlError{"Read message after sending Version", err}
@@ -190,13 +199,13 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 	// send verack command
 	msgverack := wire.NewMsgVerAck()
 
-	err = wire.WriteMessage(conn, msgverack, pver, twistNet)
+	err = wire.WriteMessage(conn, msgverack, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"writing message VerAck", err}
 	}
 
 	// second message received should be verack
-	msg, _, err = wire.ReadMessage(conn, pver, twistNet)
+	msg, _, err = wire.ReadMessage(conn, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"reading expected Ver Ack from remote client", err}
 	}
@@ -213,7 +222,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 	// send getaddr command
 	msgGetAddr := wire.NewMsgGetAddr()
 
-	err = wire.WriteMessage(conn, msgGetAddr, pver, twistNet)
+	err = wire.WriteMessage(conn, msgGetAddr, pver, netID)
 	if err != nil {
 		return nil, &crawlError{"writing Addr message to remote client", err}
 	}
@@ -225,7 +234,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
 		// Using the Bitcoin lib for the Twister Net means it does not understand some
 		// of the commands and will error. We can ignore these as we are only
 		// interested in the addr message and its content.
-		msgaddr, _, _ := wire.ReadMessage(conn, pver, twistNet)
+		msgaddr, _, _ := wire.ReadMessage(conn, pver, netID)
 		if msgaddr != nil {
 			switch msg := msgaddr.(type) {
 			case *wire.MsgAddr:

dns.go

@@ -51,7 +51,7 @@ func updateDNS(s *dnsseeder) {
 			if t == dnsV4Std || t == dnsV4Non {
 				if t == dnsV4Std && tw.dnsType == dnsV4Std {
 					r := new(dns.A)
-					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.na.IP
 					rr4std = append(rr4std, r)
 					numRR++
@@ -60,12 +60,12 @@ func updateDNS(s *dnsseeder) {
 				// if the twistee is using a non standard port then add the encoded port info to DNS
 				if t == dnsV4Non && tw.dnsType == dnsV4Non {
 					r := new(dns.A)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.na.IP
 					rr4non = append(rr4non, r)
 					numRR++
 					r = new(dns.A)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.A = tw.nonstdIP
 					rr4non = append(rr4non, r)
 					numRR++
@@ -74,7 +74,7 @@ func updateDNS(s *dnsseeder) {
 			if t == dnsV6Std || t == dnsV6Non {
 				if t == dnsV6Std && tw.dnsType == dnsV6Std {
 					r := new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.na.IP
 					rr6std = append(rr6std, r)
 					numRR++
@@ -82,12 +82,12 @@ func updateDNS(s *dnsseeder) {
 				// if the twistee is using a non standard port then add the encoded port info to DNS
 				if t == dnsV6Non && tw.dnsType == dnsV6Non {
 					r := new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.na.IP
 					rr6non = append(rr6non, r)
 					numRR++
 					r = new(dns.AAAA)
-					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
+					r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
 					r.AAAA = tw.nonstdIP
 					rr6non = append(rr6non, r)
 					numRR++
@@ -156,8 +156,8 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
 
 	w.WriteMsg(m)
 
-	if config.verbose {
+	if config.debug {
-		log.Printf("status - DNS response Type: standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
+		log.Printf("debug - DNS response Type: standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
 	}
 }
 
@@ -195,8 +195,8 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
 
 	w.WriteMsg(m)
 
-	if config.verbose {
+	if config.debug {
-		log.Printf("status - DNS response Type: non-standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
+		log.Printf("debug - DNS response Type: non-standard  To IP: %s  Query Type: %s\n", w.RemoteAddr().String(), qtype)
 	}
 }
 

doc.go

@@ -1,8 +1,17 @@
 /*
-This application provides a seeder service to the Twister Network.
+This application provides a DNS seeder service to network based on Bitcoin technology.
+For example -
 http://twister.net.co/
+https://bitcoin.org/
 
-It crawls the Twister Network for active clients and records their ip address and port. It then replies to DNS queries with the ip addresses.
+
+This application crawls the Network for active clients and records their ip address and port. It then replies to DNS queries with this information.
+
+Features:
+- Preconfigured support for Twister & Bitcoin networks. use -net <network> to load config data.
+- supports ipv4 & ipv6 addresses
+- revisits clients on a configurable time basis to make sure they are still available
+- Low memory & cpu requirements
 
 */
 package main

http.go

@@ -231,9 +231,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
 		}
 
 	}
-
 	writeFooter(w, r, startT)
-
 }
 
 // copy Twistee details into a template friendly struct

main.go

@@ -38,29 +38,48 @@ type configData struct {
 
 var config configData
 var counts twCounts
+var nwname string
 
 func main() {
 
 	// FIXME - update with git hash during build
-	config.version = "0.5.0"
+	config.version = "0.6.0"
 
 	// initialize the stats counters
 	counts.TwStatus = make([]uint32, maxStatusTypes)
 	counts.TwStarts = make([]uint32, maxStatusTypes)
 	counts.DNSCounts = make([]uint32, maxDNSTypes)
 
+	flag.StringVar(&nwname, "net", "", "Preconfigured Network config")
 	flag.StringVar(&config.host, "h", "", "DNS host to serve")
-	flag.StringVar(&config.port, "p", "8053", "Port to listen on")
+	flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on")
+	flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
 	flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
 	flag.BoolVar(&config.debug, "d", false, "Display debug output")
 	flag.BoolVar(&config.stats, "s", false, "Display stats output")
-	flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
 	flag.Parse()
 
 	if config.host == "" {
-		log.Fatalf("error - no hostname provided\n")
+		fmt.Printf("error - no hostname provided\n")
+		os.Exit(1)
 	}
 
+	// configure the network options so we can start crawling
+	thenet := selectNetwork(nwname)
+	if thenet == nil {
+		fmt.Printf("Error - No valid network specified. Please add -net=<network> from one of the following:\n")
+		for _, n := range getNetworkNames() {
+			fmt.Printf("%s\n", n)
+		}
+		os.Exit(1)
+	}
+
+	// init the seeder
+	config.seeder = &dnsseeder{}
+	config.seeder.theList = make(map[string]*twistee)
+	config.seeder.uptime = time.Now()
+	config.seeder.net = thenet
+
 	if config.debug == true {
 		config.verbose = true
 		config.stats = true
@@ -73,18 +92,14 @@ func main() {
 
 	if config.verbose == false {
 		log.Printf("status - Running in quiet mode with limited output produced\n")
+	} else {
+		log.Printf("status - system is configured for %s\n", config.seeder.net.name)
 	}
 
 	// start the web interface if we want it running
 	if config.http != "" {
 		go startHTTP(config.http)
 	}
-
-	// init the seeder
-	config.seeder = &dnsseeder{}
-	config.seeder.theList = make(map[string]*twistee)
-	config.seeder.uptime = time.Now()
-
 	// start dns server
 	dns.HandleFunc("nonstd."+config.host, handleDNSNon)
 	dns.HandleFunc(config.host, handleDNSStd)
@@ -92,7 +107,7 @@ func main() {
 	//go serve("tcp", config.port)
 
 	// seed the seeder with some ip addresses
-	initCrawlers()
+	config.seeder.initCrawlers()
 	// start first crawl
 	config.seeder.startCrawlers()
 
@@ -100,11 +115,11 @@ func main() {
 	signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
 
 	// extract good dns records from all twistees on regular basis
-	dnsChan := time.NewTicker(time.Second * 57).C
+	dnsChan := time.NewTicker(time.Second * dnsDelay).C
 	// used to start crawlers on a regular basis
-	crawlChan := time.NewTicker(time.Second * 22).C
+	crawlChan := time.NewTicker(time.Second * crawlDelay).C
 	// used to remove old statusNG twistees that have reached fail count
-	auditChan := time.NewTicker(time.Hour * 1).C
+	auditChan := time.NewTicker(time.Minute * auditDelay).C
 
 	dowhile := true
 	for dowhile == true {
@@ -115,7 +130,7 @@ func main() {
 			if config.debug {
 				log.Printf("debug - Audit twistees timer triggered\n")
 			}
-			config.seeder.auditTwistees()
+			config.seeder.auditClients()
 		case <-dnsChan:
 			if config.debug {
 				log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n")

network.go

@@ -0,0 +1,75 @@
+package main
+
+import (
+	"github.com/btcsuite/btcd/wire"
+)
+
+// network struct holds config details for the network the seeder is using
+type network struct {
+	id          wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
+	maxSize     int             // max number of clients before we start restricting new entries
+	port        uint16          // default network port this network uses
+	pver        uint32          // minimum block height for the network
+	ttl         uint32          // DNS TTL to use for this network
+	name        string          // Short name for the network
+	description string          // Long description for the network
+	seeders     []string        // slice of seeders to pull ip addresses when starting this seeder
+	maxStart    []uint32        // max number of goroutines to start each run for each status type
+	delay       []int64         // number of seconds to wait before we connect to a known client for each status
+}
+
+// getNetworkNames returns a slice of the networks that have been configured
+func getNetworkNames() []string {
+	return []string{"twister", "bitcoin", "bitcoin-testnet"}
+}
+
+// selectNetwork will return a network struct for a given network
+func selectNetwork(name string) *network {
+	switch name {
+	case "twister":
+		return &network{
+			id:          0xd2bbdaf0,
+			port:        28333,
+			pver:        60000,
+			ttl:         600,
+			maxSize:     1000,
+			name:        "TwisterNet",
+			description: "Twister P2P Net",
+			seeders:     []string{"seed2.twister.net.co", "seed.twister.net.co", "seed3.twister.net.co"},
+			maxStart:    []uint32{15, 15, 15, 30},
+			delay:       []int64{184, 678, 237, 1876},
+		}
+	case "bitcoin":
+		return &network{
+			id:          0xd9b4bef9,
+			port:        8333,
+			pver:        70001,
+			ttl:         900,
+			maxSize:     1250,
+			name:        "BitcoinMainNet",
+			description: "Bitcoin Main Net",
+			seeders:     []string{"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be"},
+			maxStart:    []uint32{20, 20, 20, 30},
+			delay:       []int64{210, 789, 234, 1876},
+		}
+	case "bitcoin-testnet":
+		return &network{
+			id:          0xdab5bffa,
+			port:        18333,
+			pver:        70001,
+			ttl:         300,
+			maxSize:     250,
+			name:        "BitcoinTestNet",
+			description: "Bitcoin Test Net",
+			seeders:     []string{"testnet-seed.alexykot.me", "testnet-seed.bitcoin.petertodd.org", "testnet-seed.bluematt.me", "testnet-seed.bitcoin.schildbach.de"},
+			maxStart:    []uint32{15, 15, 15, 30},
+			delay:       []int64{184, 678, 237, 1876},
+		}
+	default:
+		return nil
+	}
+}
+
+/*
+
+ */

seeder.go

@@ -12,68 +12,73 @@ import (
 
 const (
 
-	// TWISTNET Magic number to make it incompatible with the Bitcoin network
-	twistNet = 0xd2bbdaf0
 	// NOUNCE is used to check if we connect to ourselves
 	// as we don't listen we can use a fixed value
 	nounce  = 0x0539a019ca550825
-	pver    = 60000
 	minPort = 0
 	maxPort = 65535
 
-	twStdPort = 28333 // standard port twister listens on
+	crawlDelay = 22 // seconds between start crawlwer ticks
+	auditDelay = 22 // minutes between audit channel ticks
+	dnsDelay   = 57 // seconds between updates to active dns record list
 
 	maxFails = 58 // max number of connect fails before we delete a twistee. Just over 24 hours(checked every 33 minutes)
 
 	maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout
+)
 
-	dnsInvalid  = 0
+const (
-	dnsV4Std    = 1
+	dnsInvalid  = iota //
-	dnsV4Non    = 2
+	dnsV4Std           // ip v4 using network standard port
-	dnsV6Std    = 3
+	dnsV4Non           // ip v4 using network non standard port
-	dnsV6Non    = 4
+	dnsV6Std           // ipv6 using network standard port
-	maxDNSTypes = 5
+	dnsV6Non           // ipv6 using network non standard port
+	maxDNSTypes        // used in main to allocate slice
+)
 
+const (
 	// twistee status
-	statusRG       = 1 // reported good status. A remote twistee has reported this ip but we have not connected
+	statusRG       = iota // reported good status. A remote twistee has reported this ip but we have not connected
-	statusCG       = 2 // confirmed good. We have connected to the twistee and received addresses
+	statusCG              // confirmed good. We have connected to the twistee and received addresses
-	statusWG       = 3 // was good. Twistee was confirmed good but now having problems
+	statusWG              // was good. Twistee was confirmed good but now having problems
-	statusNG       = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
+	statusNG              // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
-	maxStatusTypes = 5
+	maxStatusTypes        // used in main to allocate slice
 )
 
 type dnsseeder struct {
-	uptime  time.Time
+	net     *network            // network struct with config options for this network
-	theList map[string]*twistee
+	uptime  time.Time           // as the name says
+	theList map[string]*twistee // the list of current clients
 	mtx     sync.RWMutex
 }
 
 // initCrawlers needs to be run before the startCrawlers so it can get
 // a list of current ip addresses from the other seeders and therefore
 // start the crawl process
-func initCrawlers() {
+func (s *dnsseeder) initCrawlers() {
 
-	seeders := []string{"seed2.twister.net.co", "seed3.twister.net.co", "seed.twister.net.co"}
+	// get a list of permenant seeders
+	seeders := s.net.seeders
 
-	for _, seeder := range seeders {
+	for _, aseeder := range seeders {
 		c := 0
 
-		newRRs, err := net.LookupHost(seeder)
+		newRRs, err := net.LookupHost(aseeder)
 		if err != nil {
-			log.Printf("status - unable to do initial lookup to seeder %s %v\n", seeder, err)
+			log.Printf("status - unable to do initial lookup to seeder %s %v\n", aseeder, err)
 			continue
 		}
 
 		for _, ip := range newRRs {
 			if newIP := net.ParseIP(ip); newIP != nil {
 				// 1 at the end is the services flag
-				if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, 28333, 1)); x == true {
+				if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, s.net.port, 1)); x == true {
 					c++
 				}
 			}
 		}
 		if config.verbose {
-			log.Printf("status - completed import of %v addresses from %s\n", c, seeder)
+			log.Printf("status - completed import of %v addresses from %s\n", c, aseeder)
 		}
 	}
 }
@@ -99,19 +104,20 @@ func (s *dnsseeder) startCrawlers() {
 		started    uint32 // count of goroutines started for this type
 		delay      int64  // number of second since last try
 	}{
-		{"statusRG", statusRG, 10, 0, 0, 184},
+		{"statusRG", statusRG, s.net.maxStart[statusRG], 0, 0, s.net.delay[statusRG]},
-		{"statusCG", statusCG, 10, 0, 0, 325},
+		{"statusCG", statusCG, s.net.maxStart[statusCG], 0, 0, s.net.delay[statusCG]},
-		{"statusWG", statusWG, 10, 0, 0, 237},
+		{"statusWG", statusWG, s.net.maxStart[statusWG], 0, 0, s.net.delay[statusWG]},
-		{"statusNG", statusNG, 20, 0, 0, 1876},
+		{"statusNG", statusNG, s.net.maxStart[statusNG], 0, 0, s.net.delay[statusNG]},
 	}
 
 	s.mtx.RLock()
 	defer s.mtx.RUnlock()
 
+	// step through each of the status types RG, CG, WG, NG
 	for _, c := range crawlers {
 
 		// range on a map will not return items in the same order each time
-		// not the best method to randomly pick twistees to crawl. FIXME
+		// so this is a random'ish selection
 		for _, tw := range s.theList {
 
 			if tw.status != c.status {
@@ -134,7 +140,7 @@ func (s *dnsseeder) startCrawlers() {
 			}
 
 			// all looks good so start a go routine to crawl the remote twistee
-			go crawlTwistee(tw)
+			go crawlTwistee(s, tw)
 			c.started++
 		}
 
@@ -166,6 +172,12 @@ func (s *dnsseeder) isNaDup(na *wire.NetAddress) bool {
 // addNa validates and adds a network address to theList
 func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
 
+	// as this is run in many different goroutines then they may all try and
+	// add new addresses so do a final check
+	if s.isFull() {
+		return false
+	}
+
 	if dup := s.isNaDup(nNa); dup == true {
 		return false
 	}
@@ -191,7 +203,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
 	// select the dns type based on the remote address type and port
 	if x := nt.na.IP.To4(); x == nil {
 		// not ipv4
-		if nNa.Port != twStdPort {
+		if nNa.Port != s.net.port {
 			nt.dnsType = dnsV6Non
 
 			// produce the nonstdIP
@@ -202,7 +214,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
 		}
 	} else {
 		// ipv4
-		if nNa.Port != twStdPort {
+		if nNa.Port != s.net.port {
 			nt.dnsType = dnsV4Non
 
 			// force ipv4 address into a 4 byte buffer
@@ -216,8 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
 	// generate the key and add to theList
 	k := net.JoinHostPort(nNa.IP.String(), strconv.Itoa(int(nNa.Port)))
 	s.mtx.Lock()
-	// final check to make sure another twistee & goroutine has not already added this twistee
+	// final check to make sure another crawl & goroutine has not already added this client
-	// FIXME migrate to use channels
 	if _, dup := s.theList[k]; dup == false {
 		s.theList[k] = &nt
 	}
@@ -259,10 +270,20 @@ func crc16(bs []byte) uint16 {
 	return crc
 }
 
-func (s *dnsseeder) auditTwistees() {
+func (s *dnsseeder) auditClients() {
 
 	c := 0
-	log.Printf("status - Audit start. System Uptime: %s\n", time.Since(s.uptime).String())
+
+	// set this early so for this audit run all NG clients will be purged
+	// and space will be made for new, possible CG clients
+	iAmFull := s.isFull()
+
+	// cgGoal is 75% of the max statusCG clients we can crawl with the current network delay & maxStart settings.
+	// This allows us to cycle statusCG users to keep the list fresh
+	cgGoal := int(float64(float64(s.net.delay[statusCG]/crawlDelay)*float64(s.net.maxStart[statusCG])) * 0.75)
+	cgCount := 0
+
+	log.Printf("status - Audit start. statusCG Goal: %v System Uptime: %s\n", cgGoal, time.Since(s.uptime).String())
 
 	s.mtx.Lock()
 	defer s.mtx.Unlock()
@@ -280,19 +301,8 @@ func (s *dnsseeder) auditTwistees() {
 					tw.statusStr)
 			}
 		}
-		if tw.status == statusRG || tw.status == statusWG {
-			if time.Now().Unix()-tw.statusTime.Unix() >= 900 {
-				log.Printf("warning - unchanged status > 15 minutes ====\n- %s status:rating:fails %v:%v:%v last status change: %s last status: %s\n====\n",
-					k,
-					tw.status,
-					tw.rating,
-					tw.connectFails,
-					tw.statusTime.String(),
-					tw.statusStr)
-			}
-		}
 
-		// last audit task is to remove twistees that we can not connect to
+		// Audit task is to remove clients that we have not been able to connect to
 		if tw.status == statusNG && tw.connectFails > maxFails {
 			if config.verbose {
 				log.Printf("status - purging twistee %s after %v failed connections\n", k, tw.connectFails)
@@ -305,6 +315,36 @@ func (s *dnsseeder) auditTwistees() {
 			delete(s.theList, k)
 		}
 
+		// If seeder is full then remove old NG clients and fill up with possible new CG clients
+		if tw.status == statusNG && iAmFull {
+			if config.verbose {
+				log.Printf("status - seeder full purging twistee %s\n", k)
+			}
+
+			c++
+			// remove the map entry and mark the old twistee as
+			// nil so garbage collector will remove it
+			s.theList[k] = nil
+			delete(s.theList, k)
+		}
+
+		// check if we need to purge statusCG to freshen the list
+		if tw.status == statusCG {
+			if cgCount++; cgCount > cgGoal {
+				// we have enough statusCG clients so purge remaining to cycle through the list
+				if config.verbose {
+					log.Printf("status - seeder cycle statusCG - purging client %s\n", k)
+				}
+
+				c++
+				// remove the map entry and mark the old twistee as
+				// nil so garbage collector will remove it
+				s.theList[k] = nil
+				delete(s.theList, k)
+			}
+
+		}
+
 	}
 	if config.verbose {
 		log.Printf("status - Audit complete. %v twistees purged\n", c)
@@ -314,10 +354,17 @@ func (s *dnsseeder) auditTwistees() {
 
 // teatload loads the dns records with time based test data
 func (s *dnsseeder) loadDNS() {
-
 	updateDNS(s)
 }
 
+// isFull returns true if the number of remote clients is more than we want to store
+func (s *dnsseeder) isFull() bool {
+	if len(s.theList) > s.net.maxSize {
+		return true
+	}
+	return false
+}
+
 /*
 
  */

twistee.go

@@ -9,22 +9,22 @@ import (
 
 // Twistee struct contains details on one twister client
 type twistee struct {
-	na           *wire.NetAddress
+	na           *wire.NetAddress // holds ip address & port details
-	lastConnect  time.Time
+	lastConnect  time.Time        // last time we sucessfully connected to this client
-	lastTry      time.Time
+	lastTry      time.Time        // last time we tried to connect to this client
-	crawlStart   time.Time
+	crawlStart   time.Time        // time when we started the last crawl
-	statusTime   time.Time
+	statusTime   time.Time        // time the status was last updated
-	crawlActive  bool
+	crawlActive  bool             // are we currently crawling this client
-	connectFails uint32
+	connectFails uint32           // number of times we have failed to connect to this client
 	statusStr    string           // string with last error or OK details
 	version      int32            // remote client protocol version
 	strVersion   string           // remote client user agent
 	services     wire.ServiceFlag // remote client supported services
-	lastBlock    int32
+	lastBlock    int32            // remote client last block
-	status       uint32 // rg,cg,wg,ng
+	status       uint32           // rg,cg,wg,ng
-	rating       uint32 // if it reaches 100 then we ban them
+	rating       uint32           // if it reaches 100 then we mark them statusNG
-	nonstdIP     net.IP
+	nonstdIP     net.IP           // if not using the default port then this is the encoded ip containing the actual port
-	dnsType      uint32
+	dnsType      uint32           // what dns type this client is
 }
 
 // status2str will return the string description of the status