twisterarmy/dnsseeder
1
0
Fork 0
mirror of https://github.com/twisterarmy/dnsseeder.git synced 2025-08-26 05:31:56 +00:00
Code Issues Projects Releases Wiki Activity

Initial support for seeding multiple networks and improve scaling for larger networks

Browse Source
This commit is contained in:
Lyndsay Roger 2015-09-02 15:09:15 +12:00
parent cf764901a1
commit 8ce82dda73
9 changed files with 270 additions and 118 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

19
README.md
View File

@ -1,15 +1,13 @@
# dnsseeder
Go Language dns seeder for the Twister P2P network
This is a dns seeder for the [Twister P2P network](http://twister.net.co/)
Go Language dns seeder for Networks that use Bitcoin technology such as the [Twister P2P network](http://twister.net.co/)
It is based on the original twister-seeder https://github.com/miguelfreitas/twister-seeder
This codebase can now seed different networks. At the moment it supports Twister and Bitcoin networks. These are configured in network.go and selected at runtime with -net command line option.
Also see the associated utility to display information about [non-standard ip addresses](https://github.com/gombadi/nonstd/)
> **NOTE:** This repository is under ongoing development. Stable releases have been tagged and should be used for production systems.
## Installing
@ -36,7 +34,7 @@ The binary will then be available in ${HOME}/go/bin
## Usage
$ dnsseeder -h <domain respond to>
$ dnsseeder -h <domain respond to> -net <network to seed>
An easy way to run the program is with tmux or screen. This enables you to log out and leave the program running.
@ -48,7 +46,8 @@ If you want to be able to view the web interface then add -w port for the web se
Command line Options:
-h hostname to serve
-p port to listen on
-net The network to seed. Currently twister, bitcoin, bitcoin-test
-p port to listen on for DNS requests
-d Produce debug output
-v Produce verbose output
-w Port to listen on for Web Interface
@ -69,16 +68,16 @@ gzip ${LOGDIR}/*.log
# pass through the logging level needed
if [ -z ${1} ]; then
LOGLV="-v"
THENET="twister"
else
LOGLV="${1}"
THENET="${1}"
fi
cd
echo
echo "======= Run the Go Language dnsseed ======="
echo
${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <port.to.listen.on> ${LOGLV} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
${HOME}/go/bin/dnsseeder -h <host.to.serve> -p <dns.port.to.listen.on> ${THENET -v} -w 8880 2>&1 | tee ${LOGDIR}/$(date +%F-%s)-goseeder.log
```

45
crawler.go
View File

@ -22,7 +22,7 @@ func (e *crawlError) Error() string {
// crawlTwistee runs in a goroutine, crawls the remote ip and updates the master
// list of currently active addresses
func crawlTwistee(tw *twistee) {
func crawlTwistee(s *dnsseeder, tw *twistee) {
tw.crawlActive = true
tw.crawlStart = time.Now()
@ -39,7 +39,7 @@ func crawlTwistee(tw *twistee) {
}
// connect to the remote ip and ask them for their addr list
ras, e := crawlIP(tw)
rna, e := crawlIP(s.net.pver, s.net.id, tw)
if e != nil {
// update the fact that we have not connected to this twistee
@ -50,9 +50,15 @@ func crawlTwistee(tw *twistee) {
// update the status of this failed twistee
switch tw.status {
case statusRG:
if tw.rating += 25; tw.rating > 30 {
tw.status = statusWG
// if we are full then any RG failures will skip directly to NG
if s.isFull() {
tw.status = statusNG // not able to connect to this twistee so ignore
tw.statusTime = time.Now()
} else {
if tw.rating += 25; tw.rating > 30 {
tw.status = statusWG
tw.statusTime = time.Now()
}
}
case statusCG:
if tw.rating += 25; tw.rating >= 50 {
@ -60,7 +66,7 @@ func crawlTwistee(tw *twistee) {
tw.statusTime = time.Now()
}
case statusWG:
if tw.rating += 30; tw.rating >= 100 {
if tw.rating += 15; tw.rating >= 100 {
tw.status = statusNG // not able to connect to this twistee so ignore
tw.statusTime = time.Now()
}
@ -93,11 +99,14 @@ func crawlTwistee(tw *twistee) {
added := 0
// loop through all the received network addresses and add to thelist if not present
for _, na := range ras {
// a new network address so add to the system
if x := config.seeder.addNa(na); x == true {
added++
// if we are full then skip adding more possible clients
if s.isFull() == false {
// loop through all the received network addresses and add to thelist if not present
for _, na := range rna {
// a new network address so add to the system
if x := s.addNa(na); x == true {
added++
}
}
}
@ -108,7 +117,7 @@ func crawlTwistee(tw *twistee) {
tw.status,
tw.rating,
tw.connectFails,
len(ras),
len(rna),
added,
time.Since(tw.crawlStart).String(),
time.Since(cs).String())
@ -125,7 +134,7 @@ func crawlEnd(tw *twistee) {
}
// crawlIP retrievs a slice of ip addresses from a client
func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
func crawlIP(pver uint32, netID wire.BitcoinNet, tw *twistee) ([]*wire.NetAddress, *crawlError) {
ip := tw.na.IP.String()
port := strconv.Itoa(int(tw.na.Port))
@ -155,14 +164,14 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
return nil, &crawlError{"Create NewMsgVersionFromConn", err}
}
err = wire.WriteMessage(conn, msgver, pver, twistNet)
err = wire.WriteMessage(conn, msgver, pver, netID)
if err != nil {
// Log and handle the error
return nil, &crawlError{"Write Version Message", err}
}
// first message received should be version
msg, _, err := wire.ReadMessage(conn, pver, twistNet)
msg, _, err := wire.ReadMessage(conn, pver, netID)
if err != nil {
// Log and handle the error
return nil, &crawlError{"Read message after sending Version", err}
@ -190,13 +199,13 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
// send verack command
msgverack := wire.NewMsgVerAck()
err = wire.WriteMessage(conn, msgverack, pver, twistNet)
err = wire.WriteMessage(conn, msgverack, pver, netID)
if err != nil {
return nil, &crawlError{"writing message VerAck", err}
}
// second message received should be verack
msg, _, err = wire.ReadMessage(conn, pver, twistNet)
msg, _, err = wire.ReadMessage(conn, pver, netID)
if err != nil {
return nil, &crawlError{"reading expected Ver Ack from remote client", err}
}
@ -213,7 +222,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
// send getaddr command
msgGetAddr := wire.NewMsgGetAddr()
err = wire.WriteMessage(conn, msgGetAddr, pver, twistNet)
err = wire.WriteMessage(conn, msgGetAddr, pver, netID)
if err != nil {
return nil, &crawlError{"writing Addr message to remote client", err}
}
@ -225,7 +234,7 @@ func crawlIP(tw *twistee) ([]*wire.NetAddress, *crawlError) {
// Using the Bitcoin lib for the Twister Net means it does not understand some
// of the commands and will error. We can ignore these as we are only
// interested in the addr message and its content.
msgaddr, _, _ := wire.ReadMessage(conn, pver, twistNet)
msgaddr, _, _ := wire.ReadMessage(conn, pver, netID)
if msgaddr != nil {
switch msg := msgaddr.(type) {
case *wire.MsgAddr:

20
dns.go
View File

@ -51,7 +51,7 @@ func updateDNS(s *dnsseeder) {
if t == dnsV4Std || t == dnsV4Non {
if t == dnsV4Std && tw.dnsType == dnsV4Std {
r := new(dns.A)
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.na.IP
rr4std = append(rr4std, r)
numRR++
@ -60,12 +60,12 @@ func updateDNS(s *dnsseeder) {
// if the twistee is using a non standard port then add the encoded port info to DNS
if t == dnsV4Non && tw.dnsType == dnsV4Non {
r := new(dns.A)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.na.IP
rr4non = append(rr4non, r)
numRR++
r = new(dns.A)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.A = tw.nonstdIP
rr4non = append(rr4non, r)
numRR++
@ -74,7 +74,7 @@ func updateDNS(s *dnsseeder) {
if t == dnsV6Std || t == dnsV6Non {
if t == dnsV6Std && tw.dnsType == dnsV6Std {
r := new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.na.IP
rr6std = append(rr6std, r)
numRR++
@ -82,12 +82,12 @@ func updateDNS(s *dnsseeder) {
// if the twistee is using a non standard port then add the encoded port info to DNS
if t == dnsV6Non && tw.dnsType == dnsV6Non {
r := new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.na.IP
rr6non = append(rr6non, r)
numRR++
r = new(dns.AAAA)
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: 60}
r.Hdr = dns.RR_Header{Name: "nonstd." + config.host + ".", Rrtype: dns.TypeAAAA, Class: dns.ClassINET, Ttl: s.net.ttl}
r.AAAA = tw.nonstdIP
rr6non = append(rr6non, r)
numRR++
@ -156,8 +156,8 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
w.WriteMsg(m)
if config.verbose {
log.Printf("status - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
if config.debug {
log.Printf("debug - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
}
}
@ -195,8 +195,8 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
w.WriteMsg(m)
if config.verbose {
log.Printf("status - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
if config.debug {
log.Printf("debug - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
}
}

13
doc.go
View File

@ -1,8 +1,17 @@
/*
This application provides a seeder service to the Twister Network.
This application provides a DNS seeder service to network based on Bitcoin technology.
For example -
http://twister.net.co/
https://bitcoin.org/
It crawls the Twister Network for active clients and records their ip address and port. It then replies to DNS queries with the ip addresses.
This application crawls the Network for active clients and records their ip address and port. It then replies to DNS queries with this information.
Features:
- Preconfigured support for Twister & Bitcoin networks. use -net <network> to load config data.
- supports ipv4 & ipv6 addresses
- revisits clients on a configurable time basis to make sure they are still available
- Low memory & cpu requirements
*/
package main

2
http.go
View File

@ -231,9 +231,7 @@ func statusHandler(w http.ResponseWriter, r *http.Request, status uint32) {
}
}
writeFooter(w, r, startT)
}
// copy Twistee details into a template friendly struct

45
main.go
View File

@ -38,29 +38,48 @@ type configData struct {
var config configData
var counts twCounts
var nwname string
func main() {
// FIXME - update with git hash during build
config.version = "0.5.0"
config.version = "0.6.0"
// initialize the stats counters
counts.TwStatus = make([]uint32, maxStatusTypes)
counts.TwStarts = make([]uint32, maxStatusTypes)
counts.DNSCounts = make([]uint32, maxDNSTypes)
flag.StringVar(&nwname, "net", "", "Preconfigured Network config")
flag.StringVar(&config.host, "h", "", "DNS host to serve")
flag.StringVar(&config.port, "p", "8053", "Port to listen on")
flag.StringVar(&config.port, "p", "8053", "DNS Port to listen on")
flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
flag.BoolVar(&config.debug, "d", false, "Display debug output")
flag.BoolVar(&config.stats, "s", false, "Display stats output")
flag.StringVar(&config.http, "w", "", "Web Port to listen on. No port specified & no web server running")
flag.Parse()
if config.host == "" {
log.Fatalf("error - no hostname provided\n")
fmt.Printf("error - no hostname provided\n")
os.Exit(1)
}
// configure the network options so we can start crawling
thenet := selectNetwork(nwname)
if thenet == nil {
fmt.Printf("Error - No valid network specified. Please add -net=<network> from one of the following:\n")
for _, n := range getNetworkNames() {
fmt.Printf("%s\n", n)
}
os.Exit(1)
}
// init the seeder
config.seeder = &dnsseeder{}
config.seeder.theList = make(map[string]*twistee)
config.seeder.uptime = time.Now()
config.seeder.net = thenet
if config.debug == true {
config.verbose = true
config.stats = true
@ -73,18 +92,14 @@ func main() {
if config.verbose == false {
log.Printf("status - Running in quiet mode with limited output produced\n")
} else {
log.Printf("status - system is configured for %s\n", config.seeder.net.name)
}
// start the web interface if we want it running
if config.http != "" {
go startHTTP(config.http)
}
// init the seeder
config.seeder = &dnsseeder{}
config.seeder.theList = make(map[string]*twistee)
config.seeder.uptime = time.Now()
// start dns server
dns.HandleFunc("nonstd."+config.host, handleDNSNon)
dns.HandleFunc(config.host, handleDNSStd)
@ -92,7 +107,7 @@ func main() {
//go serve("tcp", config.port)
// seed the seeder with some ip addresses
initCrawlers()
config.seeder.initCrawlers()
// start first crawl
config.seeder.startCrawlers()
@ -100,11 +115,11 @@ func main() {
signal.Notify(sig, syscall.SIGINT, syscall.SIGTERM)
// extract good dns records from all twistees on regular basis
dnsChan := time.NewTicker(time.Second * 57).C
dnsChan := time.NewTicker(time.Second * dnsDelay).C
// used to start crawlers on a regular basis
crawlChan := time.NewTicker(time.Second * 22).C
crawlChan := time.NewTicker(time.Second * crawlDelay).C
// used to remove old statusNG twistees that have reached fail count
auditChan := time.NewTicker(time.Hour * 1).C
auditChan := time.NewTicker(time.Minute * auditDelay).C
dowhile := true
for dowhile == true {
@ -115,7 +130,7 @@ func main() {
if config.debug {
log.Printf("debug - Audit twistees timer triggered\n")
}
config.seeder.auditTwistees()
config.seeder.auditClients()
case <-dnsChan:
if config.debug {
log.Printf("debug - DNS - Updating latest ip addresses timer triggered\n")

75
network.go Normal file
View File

@ -0,0 +1,75 @@
package main
import (
"github.com/btcsuite/btcd/wire"
)
// network struct holds config details for the network the seeder is using
type network struct {
id wire.BitcoinNet // Magic number - Unique ID for this network. Sent in header of all messages
maxSize int // max number of clients before we start restricting new entries
port uint16 // default network port this network uses
pver uint32 // minimum block height for the network
ttl uint32 // DNS TTL to use for this network
name string // Short name for the network
description string // Long description for the network
seeders []string // slice of seeders to pull ip addresses when starting this seeder
maxStart []uint32 // max number of goroutines to start each run for each status type
delay []int64 // number of seconds to wait before we connect to a known client for each status
}
// getNetworkNames returns a slice of the networks that have been configured
func getNetworkNames() []string {
return []string{"twister", "bitcoin", "bitcoin-testnet"}
}
// selectNetwork will return a network struct for a given network
func selectNetwork(name string) *network {
switch name {
case "twister":
return &network{
id: 0xd2bbdaf0,
port: 28333,
pver: 60000,
ttl: 600,
maxSize: 1000,
name: "TwisterNet",
description: "Twister P2P Net",
seeders: []string{"seed2.twister.net.co", "seed.twister.net.co", "seed3.twister.net.co"},
maxStart: []uint32{15, 15, 15, 30},
delay: []int64{184, 678, 237, 1876},
}
case "bitcoin":
return &network{
id: 0xd9b4bef9,
port: 8333,
pver: 70001,
ttl: 900,
maxSize: 1250,
name: "BitcoinMainNet",
description: "Bitcoin Main Net",
seeders: []string{"dnsseed.bluematt.me", "bitseed.xf2.org", "dnsseed.bitcoin.dashjr.org", "seed.bitcoin.sipa.be"},
maxStart: []uint32{20, 20, 20, 30},
delay: []int64{210, 789, 234, 1876},
}
case "bitcoin-testnet":
return &network{
id: 0xdab5bffa,
port: 18333,
pver: 70001,
ttl: 300,
maxSize: 250,
name: "BitcoinTestNet",
description: "Bitcoin Test Net",
seeders: []string{"testnet-seed.alexykot.me", "testnet-seed.bitcoin.petertodd.org", "testnet-seed.bluematt.me", "testnet-seed.bitcoin.schildbach.de"},
maxStart: []uint32{15, 15, 15, 30},
delay: []int64{184, 678, 237, 1876},
}
default:
return nil
}
}
/*
*/

145
seeder.go
View File

@ -12,68 +12,73 @@ import (
const (
// TWISTNET Magic number to make it incompatible with the Bitcoin network
twistNet = 0xd2bbdaf0
// NOUNCE is used to check if we connect to ourselves
// as we don't listen we can use a fixed value
nounce = 0x0539a019ca550825
pver = 60000
minPort = 0
maxPort = 65535
twStdPort = 28333 // standard port twister listens on
crawlDelay = 22 // seconds between start crawlwer ticks
auditDelay = 22 // minutes between audit channel ticks
dnsDelay = 57 // seconds between updates to active dns record list
maxFails = 58 // max number of connect fails before we delete a twistee. Just over 24 hours(checked every 33 minutes)
maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout
)
dnsInvalid = 0
dnsV4Std = 1
dnsV4Non = 2
dnsV6Std = 3
dnsV6Non = 4
maxDNSTypes = 5
const (
dnsInvalid = iota //
dnsV4Std // ip v4 using network standard port
dnsV4Non // ip v4 using network non standard port
dnsV6Std // ipv6 using network standard port
dnsV6Non // ipv6 using network non standard port
maxDNSTypes // used in main to allocate slice
)
const (
// twistee status
statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected
statusCG = 2 // confirmed good. We have connected to the twistee and received addresses
statusWG = 3 // was good. Twistee was confirmed good but now having problems
statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
maxStatusTypes = 5
statusRG = iota // reported good status. A remote twistee has reported this ip but we have not connected
statusCG // confirmed good. We have connected to the twistee and received addresses
statusWG // was good. Twistee was confirmed good but now having problems
statusNG // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
maxStatusTypes // used in main to allocate slice
)
type dnsseeder struct {
uptime time.Time
theList map[string]*twistee
net *network // network struct with config options for this network
uptime time.Time // as the name says
theList map[string]*twistee // the list of current clients
mtx sync.RWMutex
}
// initCrawlers needs to be run before the startCrawlers so it can get
// a list of current ip addresses from the other seeders and therefore
// start the crawl process
func initCrawlers() {
func (s *dnsseeder) initCrawlers() {
seeders := []string{"seed2.twister.net.co", "seed3.twister.net.co", "seed.twister.net.co"}
// get a list of permenant seeders
seeders := s.net.seeders
for _, seeder := range seeders {
for _, aseeder := range seeders {
c := 0
newRRs, err := net.LookupHost(seeder)
newRRs, err := net.LookupHost(aseeder)
if err != nil {
log.Printf("status - unable to do initial lookup to seeder %s %v\n", seeder, err)
log.Printf("status - unable to do initial lookup to seeder %s %v\n", aseeder, err)
continue
}
for _, ip := range newRRs {
if newIP := net.ParseIP(ip); newIP != nil {
// 1 at the end is the services flag
if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, 28333, 1)); x == true {
if x := config.seeder.addNa(wire.NewNetAddressIPPort(newIP, s.net.port, 1)); x == true {
c++
}
}
}
if config.verbose {
log.Printf("status - completed import of %v addresses from %s\n", c, seeder)
log.Printf("status - completed import of %v addresses from %s\n", c, aseeder)
}
}
}
@ -99,19 +104,20 @@ func (s *dnsseeder) startCrawlers() {
started uint32 // count of goroutines started for this type
delay int64 // number of second since last try
}{
{"statusRG", statusRG, 10, 0, 0, 184},
{"statusCG", statusCG, 10, 0, 0, 325},
{"statusWG", statusWG, 10, 0, 0, 237},
{"statusNG", statusNG, 20, 0, 0, 1876},
{"statusRG", statusRG, s.net.maxStart[statusRG], 0, 0, s.net.delay[statusRG]},
{"statusCG", statusCG, s.net.maxStart[statusCG], 0, 0, s.net.delay[statusCG]},
{"statusWG", statusWG, s.net.maxStart[statusWG], 0, 0, s.net.delay[statusWG]},
{"statusNG", statusNG, s.net.maxStart[statusNG], 0, 0, s.net.delay[statusNG]},
}
s.mtx.RLock()
defer s.mtx.RUnlock()
// step through each of the status types RG, CG, WG, NG
for _, c := range crawlers {
// range on a map will not return items in the same order each time
// not the best method to randomly pick twistees to crawl. FIXME
// so this is a random'ish selection
for _, tw := range s.theList {
if tw.status != c.status {
@ -134,7 +140,7 @@ func (s *dnsseeder) startCrawlers() {
}
// all looks good so start a go routine to crawl the remote twistee
go crawlTwistee(tw)
go crawlTwistee(s, tw)
c.started++
}
@ -166,6 +172,12 @@ func (s *dnsseeder) isNaDup(na *wire.NetAddress) bool {
// addNa validates and adds a network address to theList
func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
// as this is run in many different goroutines then they may all try and
// add new addresses so do a final check
if s.isFull() {
return false
}
if dup := s.isNaDup(nNa); dup == true {
return false
}
@ -191,7 +203,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
// select the dns type based on the remote address type and port
if x := nt.na.IP.To4(); x == nil {
// not ipv4
if nNa.Port != twStdPort {
if nNa.Port != s.net.port {
nt.dnsType = dnsV6Non
// produce the nonstdIP
@ -202,7 +214,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
}
} else {
// ipv4
if nNa.Port != twStdPort {
if nNa.Port != s.net.port {
nt.dnsType = dnsV4Non
// force ipv4 address into a 4 byte buffer
@ -216,8 +228,7 @@ func (s *dnsseeder) addNa(nNa *wire.NetAddress) bool {
// generate the key and add to theList
k := net.JoinHostPort(nNa.IP.String(), strconv.Itoa(int(nNa.Port)))
s.mtx.Lock()
// final check to make sure another twistee & goroutine has not already added this twistee
// FIXME migrate to use channels
// final check to make sure another crawl & goroutine has not already added this client
if _, dup := s.theList[k]; dup == false {
s.theList[k] = &nt
}
@ -259,10 +270,20 @@ func crc16(bs []byte) uint16 {
return crc
}
func (s *dnsseeder) auditTwistees() {
func (s *dnsseeder) auditClients() {
c := 0
log.Printf("status - Audit start. System Uptime: %s\n", time.Since(s.uptime).String())
// set this early so for this audit run all NG clients will be purged
// and space will be made for new, possible CG clients
iAmFull := s.isFull()
// cgGoal is 75% of the max statusCG clients we can crawl with the current network delay & maxStart settings.
// This allows us to cycle statusCG users to keep the list fresh
cgGoal := int(float64(float64(s.net.delay[statusCG]/crawlDelay)*float64(s.net.maxStart[statusCG])) * 0.75)
cgCount := 0
log.Printf("status - Audit start. statusCG Goal: %v System Uptime: %s\n", cgGoal, time.Since(s.uptime).String())
s.mtx.Lock()
defer s.mtx.Unlock()
@ -280,19 +301,8 @@ func (s *dnsseeder) auditTwistees() {
tw.statusStr)
}
}
if tw.status == statusRG || tw.status == statusWG {
if time.Now().Unix()-tw.statusTime.Unix() >= 900 {
log.Printf("warning - unchanged status > 15 minutes ====\n- %s status:rating:fails %v:%v:%v last status change: %s last status: %s\n====\n",
k,
tw.status,
tw.rating,
tw.connectFails,
tw.statusTime.String(),
tw.statusStr)
}
}
// last audit task is to remove twistees that we can not connect to
// Audit task is to remove clients that we have not been able to connect to
if tw.status == statusNG && tw.connectFails > maxFails {
if config.verbose {
log.Printf("status - purging twistee %s after %v failed connections\n", k, tw.connectFails)
@ -305,6 +315,36 @@ func (s *dnsseeder) auditTwistees() {
delete(s.theList, k)
}
// If seeder is full then remove old NG clients and fill up with possible new CG clients
if tw.status == statusNG && iAmFull {
if config.verbose {
log.Printf("status - seeder full purging twistee %s\n", k)
}
c++
// remove the map entry and mark the old twistee as
// nil so garbage collector will remove it
s.theList[k] = nil
delete(s.theList, k)
}
// check if we need to purge statusCG to freshen the list
if tw.status == statusCG {
if cgCount++; cgCount > cgGoal {
// we have enough statusCG clients so purge remaining to cycle through the list
if config.verbose {
log.Printf("status - seeder cycle statusCG - purging client %s\n", k)
}
c++
// remove the map entry and mark the old twistee as
// nil so garbage collector will remove it
s.theList[k] = nil
delete(s.theList, k)
}
}
}
if config.verbose {
log.Printf("status - Audit complete. %v twistees purged\n", c)
@ -314,10 +354,17 @@ func (s *dnsseeder) auditTwistees() {
// teatload loads the dns records with time based test data
func (s *dnsseeder) loadDNS() {
updateDNS(s)
}
// isFull returns true if the number of remote clients is more than we want to store
func (s *dnsseeder) isFull() bool {
if len(s.theList) > s.net.maxSize {
return true
}
return false
}
/*
*/

24
twistee.go
View File

@ -9,22 +9,22 @@ import (
// Twistee struct contains details on one twister client
type twistee struct {
na *wire.NetAddress
lastConnect time.Time
lastTry time.Time
crawlStart time.Time
statusTime time.Time
crawlActive bool
connectFails uint32
na *wire.NetAddress // holds ip address & port details
lastConnect time.Time // last time we sucessfully connected to this client
lastTry time.Time // last time we tried to connect to this client
crawlStart time.Time // time when we started the last crawl
statusTime time.Time // time the status was last updated
crawlActive bool // are we currently crawling this client
connectFails uint32 // number of times we have failed to connect to this client
statusStr string // string with last error or OK details
version int32 // remote client protocol version
strVersion string // remote client user agent
services wire.ServiceFlag // remote client supported services
lastBlock int32
status uint32 // rg,cg,wg,ng
rating uint32 // if it reaches 100 then we ban them
nonstdIP net.IP
dnsType uint32
lastBlock int32 // remote client last block
status uint32 // rg,cg,wg,ng
rating uint32 // if it reaches 100 then we mark them statusNG
nonstdIP net.IP // if not using the default port then this is the encoded ip containing the actual port
dnsType uint32 // what dns type this client is
}
// status2str will return the string description of the status