twisterarmy
/
dnsseeder
mirror of https://github.com/twisterarmy/dnsseeder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Performance improvements

master
Lyndsay Roger 9 years ago
parent
eebd6097a7
commit
6123b46dea
3 changed files with 32 additions and 58 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 51
      crawler.go
  2. 1
      node.go
  3. 38
      seeder.go

51
crawler.go
Unescape View File

@ -24,15 +24,13 @@ func (e *crawlError) Error() string {
// list of currently active addresses // list of currently active addresses
func crawlNode(rc chan *result, s *dnsseeder, nd *node) { func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
// connect to the remote ip and ask them for their addr list
rna, e := crawlIP(s, nd)
res := &result{ res := &result{
nas: rna,
msg: e,
node: net.JoinHostPort(nd.na.IP.String(), strconv.Itoa(int(nd.na.Port))), node: net.JoinHostPort(nd.na.IP.String(), strconv.Itoa(int(nd.na.Port))),
} }
// connect to the remote ip and ask them for their addr list
res.nas, res.msg = crawlIP(s, res)
// all done so push the result back to the seeder. // all done so push the result back to the seeder.
//This will block until the seeder reads the result //This will block until the seeder reads the result
rc <- res rc <- res
@ -41,34 +39,19 @@ func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
} }
// crawlIP retrievs a slice of ip addresses from a client // crawlIP retrievs a slice of ip addresses from a client
func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) { func crawlIP(s *dnsseeder, r *result) ([]*wire.NetAddress, *crawlError) {
ip := nd.na.IP.String()
port := strconv.Itoa(int(nd.na.Port))
// get correct formatting for ipv6 addresses conn, err := net.DialTimeout("tcp", r.node, time.Second*10)
dialString := net.JoinHostPort(ip, port)
if config.debug {
log.Printf("%s - debug - start crawl: node %s status: %v:%v lastcrawl: %s\n",
s.name,
dialString,
nd.status,
nd.rating,
time.Since(nd.crawlStart).String())
}
conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
if err != nil { if err != nil {
if config.debug { if config.debug {
log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, ip, err) log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, r.node, err)
} }
return nil, &crawlError{"", err} return nil, &crawlError{"", err}
} }
defer conn.Close() defer conn.Close()
if config.debug { if config.debug {
log.Printf("%s - debug - Connected to remote address: %s Last connect was %v ago\n", s.name, ip, time.Since(nd.lastConnect).String()) log.Printf("%s - debug - Connected to remote address: %s\n", s.name, r.node)
} }
// set a deadline for all comms to be done by. After this all i/o will error // set a deadline for all comms to be done by. After this all i/o will error
@ -98,17 +81,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgVersion: case *wire.MsgVersion:
// The message is a pointer to a MsgVersion struct. // The message is a pointer to a MsgVersion struct.
if config.debug { if config.debug {
log.Printf("%s - debug - %s - Remote version: %v\n", s.name, ip, msg.ProtocolVersion) log.Printf("%s - debug - %s - Remote version: %v\n", s.name, r.node, msg.ProtocolVersion)
} }
// fill the node struct with the remote details // fill the node struct with the remote details
nd.version = msg.ProtocolVersion r.version = msg.ProtocolVersion
nd.services = msg.Services r.services = msg.Services
nd.lastBlock = msg.LastBlock r.lastBlock = msg.LastBlock
if nd.strVersion != msg.UserAgent { r.strVersion = msg.UserAgent
// if the srtVersion is already the same then don't overwrite it.
// saves the GC having to cleanup a perfectly good string
nd.strVersion = msg.UserAgent
}
default: default:
return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")} return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")}
} }
@ -130,7 +109,7 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
switch msg.(type) { switch msg.(type) {
case *wire.MsgVerAck: case *wire.MsgVerAck:
if config.debug { if config.debug {
log.Printf("%s - debug - %s - received Version Ack\n", s.name, ip) log.Printf("%s - debug - %s - received Version Ack\n", s.name, r.node)
} }
default: default:
return nil, &crawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")} return nil, &crawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")}
@ -162,13 +141,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgAddr: case *wire.MsgAddr:
// received the addr message so return the result // received the addr message so return the result
if config.debug { if config.debug {
log.Printf("%s - debug - %s - received valid addr message\n", s.name, ip) log.Printf("%s - debug - %s - received valid addr message\n", s.name, r.node)
} }
dowhile = false dowhile = false
return msg.AddrList, nil return msg.AddrList, nil
default: default:
if config.debug { if config.debug {
log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, ip, msg.Command()) log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, r.node, msg.Command())
} }
} }
} }

1
node.go
Unescape View File

@ -13,7 +13,6 @@ type node struct {
lastConnect time.Time // last time we sucessfully connected to this client lastConnect time.Time // last time we sucessfully connected to this client
lastTry time.Time // last time we tried to connect to this client lastTry time.Time // last time we tried to connect to this client
crawlStart time.Time // time when we started the last crawl crawlStart time.Time // time when we started the last crawl
statusTime time.Time // time the status was last updated
nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port
statusStr string // string with last error or OK details statusStr string // string with last error or OK details
strVersion string // remote client user agent strVersion string // remote client user agent

38
seeder.go
Unescape View File

@ -50,10 +50,6 @@ type dnsseeder struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
theList map[string]*node // the list of current nodes theList map[string]*node // the list of current nodes
mtx sync.RWMutex // protect thelist mtx sync.RWMutex // protect thelist
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
dnsHost string // dns host we will serve results for this domain dnsHost string // dns host we will serve results for this domain
name string // Short name for the network name string // Short name for the network
desc string // Long description for the network desc string // Long description for the network
@ -62,12 +58,20 @@ type dnsseeder struct {
maxStart []uint32 // max number of goroutines to start each run for each status type maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status delay []int64 // number of seconds to wait before we connect to a known client for each status
counts NodeCounts // structure to hold stats for this seeder counts NodeCounts // structure to hold stats for this seeder
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
} }
type result struct { type result struct {
nas []*wire.NetAddress // slice of node addresses returned from a node nas []*wire.NetAddress // slice of node addresses returned from a node
msg *crawlError // error string or nil if no problems msg *crawlError // error string or nil if no problems
node string // theList key to the node that was crawled node string // theList key to the node that was crawled
version int32 // remote node protocol version
services wire.ServiceFlag // remote client supported services
lastBlock int32 // last block seen by the node
strVersion string // remote client user agent
} }
// initCrawlers needs to be run before the startCrawlers so it can get // initCrawlers needs to be run before the startCrawlers so it can get
@ -173,7 +177,7 @@ func (s *dnsseeder) startCrawlers(resultsChan chan *result) {
tcount := uint32(len(s.theList)) tcount := uint32(len(s.theList))
if tcount == 0 { if tcount == 0 {
if config.debug { if config.debug {
log.Printf("%s - debug - startCrawlers fail: no node ailable\n", s.name) log.Printf("%s - debug - startCrawlers fail: no node available\n", s.name)
} }
return return
} }
@ -247,22 +251,18 @@ func (s *dnsseeder) processResult(r *result) {
// if we are full then any RG failures will skip directly to NG // if we are full then any RG failures will skip directly to NG
if len(s.theList) > s.maxSize { if len(s.theList) > s.maxSize {
nd.status = statusNG // not able to connect to this node so ignore nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
} else { } else {
if nd.rating += 25; nd.rating > 30 { if nd.rating += 25; nd.rating > 30 {
nd.status = statusWG nd.status = statusWG
nd.statusTime = time.Now()
} }
} }
case statusCG: case statusCG:
if nd.rating += 25; nd.rating >= 50 { if nd.rating += 25; nd.rating >= 50 {
nd.status = statusWG nd.status = statusWG
nd.statusTime = time.Now()
} }
case statusWG: case statusWG:
if nd.rating += 15; nd.rating >= 100 { if nd.rating += 15; nd.rating >= 100 {
nd.status = statusNG // not able to connect to this node so ignore nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
} }
} }
// no more to do so return which will shutdown the goroutine & call // no more to do so return which will shutdown the goroutine & call
@ -281,23 +281,21 @@ func (s *dnsseeder) processResult(r *result) {
} }
// succesful connection and addresses received so mark status // succesful connection and addresses received so mark status
if nd.status != statusCG { nd.status = statusCG
nd.status = statusCG
nd.statusTime = time.Now()
}
cs := nd.lastConnect cs := nd.lastConnect
nd.rating = 0 nd.rating = 0
nd.connectFails = 0 nd.connectFails = 0
nd.lastConnect = time.Now() nd.lastConnect = time.Now()
nd.lastTry = time.Now() nd.lastTry = nd.lastConnect
nd.statusStr = "ok: received remote address list" nd.statusStr = "ok: received remote address list"
added := 0 added := 0
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))
// if we are full then skip adding more possible clients // if we are full then skip adding more possible clients
if len(s.theList) < s.maxSize { if len(s.theList) < s.maxSize {
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))
// loop through all the received network addresses and add to thelist if not present // loop through all the received network addresses and add to thelist if not present
for _, na := range r.nas { for _, na := range r.nas {
// a new network address so add to the system // a new network address so add to the system
@ -322,7 +320,6 @@ func (s *dnsseeder) processResult(r *result) {
time.Since(nd.crawlStart).String(), time.Since(nd.crawlStart).String(),
time.Since(cs).String()) time.Since(cs).String())
} }
} }
// crawlEnd is run as a defer to make sure node status is correctly updated // crawlEnd is run as a defer to make sure node status is correctly updated
@ -358,7 +355,6 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
lastConnect: time.Now(), lastConnect: time.Now(),
version: 0, version: 0,
status: statusRG, status: statusRG,
statusTime: time.Now(),
dnsType: dnsV4Std, dnsType: dnsV4Std,
} }

Write Preview
Loading…
Cancel
Save