Performance improvements

crawler.go

@@ -24,15 +24,13 @@ func (e *crawlError) Error() string {
 // list of currently active addresses
 func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
 
-	// connect to the remote ip and ask them for their addr list
-	rna, e := crawlIP(s, nd)
-
 	res := &result{
-		nas:  rna,
-		msg:  e,
 		node: net.JoinHostPort(nd.na.IP.String(), strconv.Itoa(int(nd.na.Port))),
 	}
 
+	// connect to the remote ip and ask them for their addr list
+	res.nas, res.msg = crawlIP(s, res)
+
 	// all done so push the result back to the seeder.
 	//This will block until the seeder reads the result
 	rc <- res
@@ -41,34 +39,19 @@ func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
 }
 
 // crawlIP retrievs a slice of ip addresses from a client
-func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
+func crawlIP(s *dnsseeder, r *result) ([]*wire.NetAddress, *crawlError) {
 
-	ip := nd.na.IP.String()
+	conn, err := net.DialTimeout("tcp", r.node, time.Second*10)
-	port := strconv.Itoa(int(nd.na.Port))
-
-	// get correct formatting for ipv6 addresses
-	dialString := net.JoinHostPort(ip, port)
-
-	if config.debug {
-		log.Printf("%s - debug - start crawl: node %s status: %v:%v lastcrawl: %s\n",
-			s.name,
-			dialString,
-			nd.status,
-			nd.rating,
-			time.Since(nd.crawlStart).String())
-	}
-
-	conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
 	if err != nil {
 		if config.debug {
-			log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, ip, err)
+			log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, r.node, err)
 		}
 		return nil, &crawlError{"", err}
 	}
 
 	defer conn.Close()
 	if config.debug {
-		log.Printf("%s - debug - Connected to remote address: %s Last connect was %v ago\n", s.name, ip, time.Since(nd.lastConnect).String())
+		log.Printf("%s - debug - Connected to remote address: %s\n", s.name, r.node)
 	}
 
 	// set a deadline for all comms to be done by. After this all i/o will error
@@ -98,17 +81,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
 	case *wire.MsgVersion:
 		// The message is a pointer to a MsgVersion struct.
 		if config.debug {
-			log.Printf("%s - debug - %s - Remote version: %v\n", s.name, ip, msg.ProtocolVersion)
+			log.Printf("%s - debug - %s - Remote version: %v\n", s.name, r.node, msg.ProtocolVersion)
 		}
 		// fill the node struct with the remote details
-		nd.version = msg.ProtocolVersion
+		r.version = msg.ProtocolVersion
-		nd.services = msg.Services
+		r.services = msg.Services
-		nd.lastBlock = msg.LastBlock
+		r.lastBlock = msg.LastBlock
-		if nd.strVersion != msg.UserAgent {
+		r.strVersion = msg.UserAgent
-			// if the srtVersion is already the same then don't overwrite it.
-			// saves the GC having to cleanup a perfectly good string
-			nd.strVersion = msg.UserAgent
-		}
 	default:
 		return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")}
 	}
@@ -130,7 +109,7 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
 	switch msg.(type) {
 	case *wire.MsgVerAck:
 		if config.debug {
-			log.Printf("%s - debug - %s - received Version Ack\n", s.name, ip)
+			log.Printf("%s - debug - %s - received Version Ack\n", s.name, r.node)
 		}
 	default:
 		return nil, &crawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")}
@@ -162,13 +141,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
 			case *wire.MsgAddr:
 				// received the addr message so return the result
 				if config.debug {
-					log.Printf("%s - debug - %s - received valid addr message\n", s.name, ip)
+					log.Printf("%s - debug - %s - received valid addr message\n", s.name, r.node)
 				}
 				dowhile = false
 				return msg.AddrList, nil
 			default:
 				if config.debug {
-					log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, ip, msg.Command())
+					log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, r.node, msg.Command())
 				}
 			}
 		}

node.go

@@ -13,7 +13,6 @@ type node struct {
 	lastConnect  time.Time        // last time we sucessfully connected to this client
 	lastTry      time.Time        // last time we tried to connect to this client
 	crawlStart   time.Time        // time when we started the last crawl
-	statusTime   time.Time        // time the status was last updated
 	nonstdIP     net.IP           // if not using the default port then this is the encoded ip containing the actual port
 	statusStr    string           // string with last error or OK details
 	strVersion   string           // remote client user agent

seeder.go

@@ -50,10 +50,6 @@ type dnsseeder struct {
 	id        wire.BitcoinNet  // Magic number - Unique ID for this network. Sent in header of all messages
 	theList   map[string]*node // the list of current nodes
 	mtx       sync.RWMutex     // protect thelist
-	maxSize   int              // max number of clients before we start restricting new entries
-	port      uint16           // default network port this seeder uses
-	pver      uint32           // minimum block height for the seeder
-	ttl       uint32           // DNS TTL to use for this seeder
 	dnsHost   string           // dns host we will serve results for this domain
 	name      string           // Short name for the network
 	desc      string           // Long description for the network
@@ -62,12 +58,20 @@ type dnsseeder struct {
 	maxStart  []uint32         // max number of goroutines to start each run for each status type
 	delay     []int64          // number of seconds to wait before we connect to a known client for each status
 	counts    NodeCounts       // structure to hold stats for this seeder
+	pver      uint32           // minimum block height for the seeder
+	ttl       uint32           // DNS TTL to use for this seeder
+	maxSize   int              // max number of clients before we start restricting new entries
+	port      uint16           // default network port this seeder uses
 }
 
 type result struct {
 	nas        []*wire.NetAddress // slice of node addresses returned from a node
 	msg        *crawlError        // error string or nil if no problems
 	node       string             // theList key to the node that was crawled
+	version    int32              // remote node protocol version
+	services   wire.ServiceFlag   // remote client supported services
+	lastBlock  int32              // last block seen by the node
+	strVersion string             // remote client user agent
 }
 
 // initCrawlers needs to be run before the startCrawlers so it can get
@@ -173,7 +177,7 @@ func (s *dnsseeder) startCrawlers(resultsChan chan *result) {
 	tcount := uint32(len(s.theList))
 	if tcount == 0 {
 		if config.debug {
-			log.Printf("%s - debug - startCrawlers fail: no node ailable\n", s.name)
+			log.Printf("%s - debug - startCrawlers fail: no node available\n", s.name)
 		}
 		return
 	}
@@ -247,22 +251,18 @@ func (s *dnsseeder) processResult(r *result) {
 			// if we are full then any RG failures will skip directly to NG
 			if len(s.theList) > s.maxSize {
 				nd.status = statusNG // not able to connect to this node so ignore
-				nd.statusTime = time.Now()
 			} else {
 				if nd.rating += 25; nd.rating > 30 {
 					nd.status = statusWG
-					nd.statusTime = time.Now()
 				}
 			}
 		case statusCG:
 			if nd.rating += 25; nd.rating >= 50 {
 				nd.status = statusWG
-				nd.statusTime = time.Now()
 			}
 		case statusWG:
 			if nd.rating += 15; nd.rating >= 100 {
 				nd.status = statusNG // not able to connect to this node so ignore
-				nd.statusTime = time.Now()
 			}
 		}
 		// no more to do so return which will shutdown the goroutine & call
@@ -281,23 +281,21 @@ func (s *dnsseeder) processResult(r *result) {
 	}
 
 	// succesful connection and addresses received so mark status
-	if nd.status != statusCG {
 	nd.status = statusCG
-		nd.statusTime = time.Now()
-	}
 	cs := nd.lastConnect
 	nd.rating = 0
 	nd.connectFails = 0
 	nd.lastConnect = time.Now()
-	nd.lastTry = time.Now()
+	nd.lastTry = nd.lastConnect
 	nd.statusStr = "ok: received remote address list"
 
 	added := 0
-	// do not accept more than one third of maxSize addresses from one node
-	oneThird := int(float64(s.maxSize / 3))
 
 	// if we are full then skip adding more possible clients
 	if len(s.theList) < s.maxSize {
+		// do not accept more than one third of maxSize addresses from one node
+		oneThird := int(float64(s.maxSize / 3))
+
 		// loop through all the received network addresses and add to thelist if not present
 		for _, na := range r.nas {
 			// a new network address so add to the system
@@ -322,7 +320,6 @@ func (s *dnsseeder) processResult(r *result) {
 			time.Since(nd.crawlStart).String(),
 			time.Since(cs).String())
 	}
-
 }
 
 // crawlEnd is run as a defer to make sure node status is correctly updated
@@ -358,7 +355,6 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
 		lastConnect: time.Now(),
 		version:     0,
 		status:      statusRG,
-		statusTime:  time.Now(),
 		dnsType:     dnsV4Std,
 	}