twisterarmy/dnsseeder
1
0
Fork 0
mirror of https://github.com/twisterarmy/dnsseeder.git synced 2025-08-26 05:31:56 +00:00
Code Issues Projects Releases Wiki Activity

Performance improvements

Browse Source
This commit is contained in:
Lyndsay Roger 2015-10-02 13:40:48 +13:00
parent eebd6097a7
commit 6123b46dea
3 changed files with 32 additions and 58 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

51
crawler.go
View File

@ -24,15 +24,13 @@ func (e *crawlError) Error() string {
// list of currently active addresses
func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
// connect to the remote ip and ask them for their addr list
rna, e := crawlIP(s, nd)
res := &result{
nas: rna,
msg: e,
node: net.JoinHostPort(nd.na.IP.String(), strconv.Itoa(int(nd.na.Port))),
}
// connect to the remote ip and ask them for their addr list
res.nas, res.msg = crawlIP(s, res)
// all done so push the result back to the seeder.
//This will block until the seeder reads the result
rc <- res
@ -41,34 +39,19 @@ func crawlNode(rc chan *result, s *dnsseeder, nd *node) {
}
// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
func crawlIP(s *dnsseeder, r *result) ([]*wire.NetAddress, *crawlError) {
ip := nd.na.IP.String()
port := strconv.Itoa(int(nd.na.Port))
// get correct formatting for ipv6 addresses
dialString := net.JoinHostPort(ip, port)
if config.debug {
log.Printf("%s - debug - start crawl: node %s status: %v:%v lastcrawl: %s\n",
s.name,
dialString,
nd.status,
nd.rating,
time.Since(nd.crawlStart).String())
}
conn, err := net.DialTimeout("tcp", dialString, time.Second*10)
conn, err := net.DialTimeout("tcp", r.node, time.Second*10)
if err != nil {
if config.debug {
log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, ip, err)
log.Printf("%s - debug - Could not connect to %s - %v\n", s.name, r.node, err)
}
return nil, &crawlError{"", err}
}
defer conn.Close()
if config.debug {
log.Printf("%s - debug - Connected to remote address: %s Last connect was %v ago\n", s.name, ip, time.Since(nd.lastConnect).String())
log.Printf("%s - debug - Connected to remote address: %s\n", s.name, r.node)
}
// set a deadline for all comms to be done by. After this all i/o will error
@ -98,17 +81,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgVersion:
// The message is a pointer to a MsgVersion struct.
if config.debug {
log.Printf("%s - debug - %s - Remote version: %v\n", s.name, ip, msg.ProtocolVersion)
log.Printf("%s - debug - %s - Remote version: %v\n", s.name, r.node, msg.ProtocolVersion)
}
// fill the node struct with the remote details
nd.version = msg.ProtocolVersion
nd.services = msg.Services
nd.lastBlock = msg.LastBlock
if nd.strVersion != msg.UserAgent {
// if the srtVersion is already the same then don't overwrite it.
// saves the GC having to cleanup a perfectly good string
nd.strVersion = msg.UserAgent
}
r.version = msg.ProtocolVersion
r.services = msg.Services
r.lastBlock = msg.LastBlock
r.strVersion = msg.UserAgent
default:
return nil, &crawlError{"Did not receive expected Version message from remote client", errors.New("")}
}
@ -130,7 +109,7 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
switch msg.(type) {
case *wire.MsgVerAck:
if config.debug {
log.Printf("%s - debug - %s - received Version Ack\n", s.name, ip)
log.Printf("%s - debug - %s - received Version Ack\n", s.name, r.node)
}
default:
return nil, &crawlError{"Did not receive expected Ver Ack message from remote client", errors.New("")}
@ -162,13 +141,13 @@ func crawlIP(s *dnsseeder, nd *node) ([]*wire.NetAddress, *crawlError) {
case *wire.MsgAddr:
// received the addr message so return the result
if config.debug {
log.Printf("%s - debug - %s - received valid addr message\n", s.name, ip)
log.Printf("%s - debug - %s - received valid addr message\n", s.name, r.node)
}
dowhile = false
return msg.AddrList, nil
default:
if config.debug {
log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, ip, msg.Command())
log.Printf("%s - debug - %s - ignoring message - %v\n", s.name, r.node, msg.Command())
}
}
}

1
node.go
View File

@ -13,7 +13,6 @@ type node struct {
lastConnect time.Time // last time we sucessfully connected to this client
lastTry time.Time // last time we tried to connect to this client
crawlStart time.Time // time when we started the last crawl
statusTime time.Time // time the status was last updated
nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port
statusStr string // string with last error or OK details
strVersion string // remote client user agent

38
seeder.go
View File

@ -50,10 +50,6 @@ type dnsseeder struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
theList map[string]*node // the list of current nodes
mtx sync.RWMutex // protect thelist
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
dnsHost string // dns host we will serve results for this domain
name string // Short name for the network
desc string // Long description for the network
@ -62,12 +58,20 @@ type dnsseeder struct {
maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status
counts NodeCounts // structure to hold stats for this seeder
pver uint32 // minimum block height for the seeder
ttl uint32 // DNS TTL to use for this seeder
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this seeder uses
}
type result struct {
nas []*wire.NetAddress // slice of node addresses returned from a node
msg *crawlError // error string or nil if no problems
node string // theList key to the node that was crawled
nas []*wire.NetAddress // slice of node addresses returned from a node
msg *crawlError // error string or nil if no problems
node string // theList key to the node that was crawled
version int32 // remote node protocol version
services wire.ServiceFlag // remote client supported services
lastBlock int32 // last block seen by the node
strVersion string // remote client user agent
}
// initCrawlers needs to be run before the startCrawlers so it can get
@ -173,7 +177,7 @@ func (s *dnsseeder) startCrawlers(resultsChan chan *result) {
tcount := uint32(len(s.theList))
if tcount == 0 {
if config.debug {
log.Printf("%s - debug - startCrawlers fail: no node ailable\n", s.name)
log.Printf("%s - debug - startCrawlers fail: no node available\n", s.name)
}
return
}
@ -247,22 +251,18 @@ func (s *dnsseeder) processResult(r *result) {
// if we are full then any RG failures will skip directly to NG
if len(s.theList) > s.maxSize {
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
} else {
if nd.rating += 25; nd.rating > 30 {
nd.status = statusWG
nd.statusTime = time.Now()
}
}
case statusCG:
if nd.rating += 25; nd.rating >= 50 {
nd.status = statusWG
nd.statusTime = time.Now()
}
case statusWG:
if nd.rating += 15; nd.rating >= 100 {
nd.status = statusNG // not able to connect to this node so ignore
nd.statusTime = time.Now()
}
}
// no more to do so return which will shutdown the goroutine & call
@ -281,23 +281,21 @@ func (s *dnsseeder) processResult(r *result) {
}
// succesful connection and addresses received so mark status
if nd.status != statusCG {
nd.status = statusCG
nd.statusTime = time.Now()
}
nd.status = statusCG
cs := nd.lastConnect
nd.rating = 0
nd.connectFails = 0
nd.lastConnect = time.Now()
nd.lastTry = time.Now()
nd.lastTry = nd.lastConnect
nd.statusStr = "ok: received remote address list"
added := 0
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))
// if we are full then skip adding more possible clients
if len(s.theList) < s.maxSize {
// do not accept more than one third of maxSize addresses from one node
oneThird := int(float64(s.maxSize / 3))
// loop through all the received network addresses and add to thelist if not present
for _, na := range r.nas {
// a new network address so add to the system
@ -322,7 +320,6 @@ func (s *dnsseeder) processResult(r *result) {
time.Since(nd.crawlStart).String(),
time.Since(cs).String())
}
}
// crawlEnd is run as a defer to make sure node status is correctly updated
@ -358,7 +355,6 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
lastConnect: time.Now(),
version: 0,
status: statusRG,
statusTime: time.Now(),
dnsType: dnsV4Std,
}