twisterarmy
/
dnsseeder
mirror of https://github.com/twisterarmy/dnsseeder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Improve global stats processing

master
Lyndsay Roger 9 years ago
parent
6a18e1aaaf
commit
b4cae52c0f
4 changed files with 107 additions and 51 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      dns.go
  2. 51
      http.go
  3. 32
      main.go
  4. 39
      seeder.go

18
dns.go
Unescape View File

@ -141,21 +141,24 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
m.Answer = getv4stdRR() m.Answer = getv4stdRR()
latest.mtx.RUnlock() latest.mtx.RUnlock()
qtype = "A" qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Std)
case dns.TypeAAAA: case dns.TypeAAAA:
latest.mtx.RLock() latest.mtx.RLock()
m.Answer = getv6stdRR() m.Answer = getv6stdRR()
latest.mtx.RUnlock() latest.mtx.RUnlock()
qtype = "AAAA" qtype = "AAAA"
go updateDNSCounts(dnsV6Std)
default: default:
// return no answer to all other queries // return no answer to all other queries
} }
w.WriteMsg(m)
if config.verbose { if config.verbose {
log.Printf("status - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) log.Printf("status - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
} }
// FIXME - add stats and query counts
w.WriteMsg(m)
} }
// handleDNSNon processes a DNS request from remote client and returns // handleDNSNon processes a DNS request from remote client and returns
@ -177,21 +180,24 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
m.Answer = getv4nonRR() m.Answer = getv4nonRR()
latest.mtx.RUnlock() latest.mtx.RUnlock()
qtype = "A" qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Non)
case dns.TypeAAAA: case dns.TypeAAAA:
latest.mtx.RLock() latest.mtx.RLock()
m.Answer = getv6nonRR() m.Answer = getv6nonRR()
latest.mtx.RUnlock() latest.mtx.RUnlock()
qtype = "AAAA" qtype = "AAAA"
go updateDNSCounts(dnsV6Non)
default: default:
// return no answer to all other queries // return no answer to all other queries
} }
w.WriteMsg(m)
if config.verbose { if config.verbose {
log.Printf("status - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype) log.Printf("status - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
} }
// FIXME - add stats and query counts
w.WriteMsg(m)
} }
// serve starts the requested DNS server listening on the requested port // serve starts the requested DNS server listening on the requested port

51
http.go
Unescape View File

@ -390,6 +390,43 @@ func generateWebStatus(status uint32) (ws []webstatus) {
// genHeader will output the standard header // genHeader will output the standard header
func writeHeader(w http.ResponseWriter, r *http.Request) { func writeHeader(w http.ResponseWriter, r *http.Request) {
var hc struct {
RG uint32
RGS uint32
CG uint32
CGS uint32
WG uint32
WGS uint32
NG uint32
NGS uint32
Total uint32
V4Std uint32
V4Non uint32
V6Std uint32
V6Non uint32
DNSTotal uint32
}
// fill the structs so they can be displayed via the template
counts.mtx.RLock()
hc.RG = counts.TwStatus[statusRG]
hc.RGS = counts.TwStarts[statusRG]
hc.CG = counts.TwStatus[statusCG]
hc.CGS = counts.TwStarts[statusCG]
hc.WG = counts.TwStatus[statusWG]
hc.WGS = counts.TwStarts[statusWG]
hc.NG = counts.TwStatus[statusNG]
hc.NGS = counts.TwStarts[statusNG]
hc.Total = hc.RG + hc.CG + hc.WG + hc.NG
hc.V4Std = counts.DNSCounts[dnsV4Std]
hc.V4Non = counts.DNSCounts[dnsV4Non]
hc.V6Std = counts.DNSCounts[dnsV6Std]
hc.V6Non = counts.DNSCounts[dnsV6Non]
hc.DNSTotal = hc.V4Std + hc.V4Non + hc.V6Std + hc.V6Non
counts.mtx.RUnlock()
// we are using basic and simple html here. No fancy graphics or css // we are using basic and simple html here. No fancy graphics or css
h := ` h := `
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd"> <!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
@ -401,22 +438,28 @@ func writeHeader(w http.ResponseWriter, r *http.Request) {
<a href="/statusNG">statusNG</a> <a href="/statusNG">statusNG</a>
<a href="/dns">DNS</a> <a href="/dns">DNS</a>
<br> <br>
Current Stats (count/started) <table><tr><td>
Twistee Stats (count/started)<br>
<table border=1><tr> <table border=1><tr>
<td>RG: {{.RG}}/{{.RGS}}</td><td>CG: {{.CG}}/{{.CGS}}</td><td>WG: {{.WG}}/{{.WGS}}</td><td>NG: {{.NG}}/{{.NGS}}</td><td>Total: {{.Total}}</td> <td>RG: {{.RG}}/{{.RGS}}</td><td>CG: {{.CG}}/{{.CGS}}</td><td>WG: {{.WG}}/{{.WGS}}</td><td>NG: {{.NG}}/{{.NGS}}</td><td>Total: {{.Total}}</td>
</tr></table> </tr></table>
</td><td>
DNS Requests<br>
<table border=1><tr>
<td>V4 Std: {{.V4Std}}</td><td>V4 Non: {{.V4Non}}</td><td>V6 Std: {{.V6Std}}</td><td>V6 Non: {{.V6Non}}</td><td>Total: {{.DNSTotal}}</td>
</tr></table>
</td></tr></table>
</center> </center>
<hr> <hr>
` `
t := template.New("Header template") t := template.New("Header template")
t, err := t.Parse(h) t, err := t.Parse(h)
if err != nil { if err != nil {
log.Printf("error parsing template %v\n", err) log.Printf("error parsing template %v\n", err)
} }
counts.mtx.RLock() err = t.Execute(w, hc)
err = t.Execute(w, counts)
counts.mtx.RUnlock()
if err != nil { if err != nil {
log.Printf("error executing template %v\n", err) log.Printf("error executing template %v\n", err)
} }

32
main.go
Unescape View File

@ -9,12 +9,22 @@ import (
"log" "log"
"os" "os"
"os/signal" "os/signal"
"sync"
"syscall" "syscall"
"time" "time"
"github.com/miekg/dns" "github.com/miekg/dns"
) )
// twCounts holds various statistics about the running system
type twCounts struct {
TwStatus []uint32
TwStarts []uint32
DNSCounts []uint32
mtx sync.RWMutex
}
// configData holds information on the application
type configData struct { type configData struct {
host string host string
port string port string
@ -34,6 +44,11 @@ func main() {
// FIXME - update with git hash during build // FIXME - update with git hash during build
config.version = "0.5.0" config.version = "0.5.0"
// initialize the stats counters
counts.TwStatus = make([]uint32, maxStatusTypes)
counts.TwStarts = make([]uint32, maxStatusTypes)
counts.DNSCounts = make([]uint32, maxDNSTypes)
flag.StringVar(&config.host, "h", "", "DNS host to serve") flag.StringVar(&config.host, "h", "", "DNS host to serve")
flag.StringVar(&config.port, "p", "8053", "Port to listen on") flag.StringVar(&config.port, "p", "8053", "Port to listen on")
flag.BoolVar(&config.verbose, "v", false, "Display verbose output") flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
@ -117,6 +132,23 @@ func main() {
fmt.Printf("\nProgram exiting. Bye\n") fmt.Printf("\nProgram exiting. Bye\n")
} }
// updateTwCounts runs in a goroutine and updates the global stats with the lates
// counts from a startCrawlers run
func updateTwCounts(status, total, started uint32) {
// update the stats counters
counts.mtx.Lock()
counts.TwStatus[status] = total
counts.TwStarts[status] = started
counts.mtx.Unlock()
}
// updateDNSCounts runs in a goroutine and updates the global stats for the number of DNS requests
func updateDNSCounts(dnsType uint32) {
counts.mtx.Lock()
counts.DNSCounts[dnsType]++
counts.mtx.Unlock()
}
/* /*
*/ */

39
seeder.go
Unescape View File

@ -27,32 +27,21 @@ const (
maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout
dnsInvalid = 0
dnsV4Std = 1 dnsV4Std = 1
dnsV4Non = 2 dnsV4Non = 2
dnsV6Std = 3 dnsV6Std = 3
dnsV6Non = 4 dnsV6Non = 4
maxDNSTypes = 5
// twistee status // twistee status
statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected
statusCG = 2 // confirmed good. We have connected to the twistee and received addresses statusCG = 2 // confirmed good. We have connected to the twistee and received addresses
statusWG = 3 // was good. Twistee was confirmed good but now having problems statusWG = 3 // was good. Twistee was confirmed good but now having problems
statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
maxStatusTypes = 5
) )
type twCounts struct {
RG uint32
RGS uint32
CG uint32
CGS uint32
WG uint32
WGS uint32
NG uint32
NGS uint32
Total int
mtx sync.RWMutex
}
type dnsseeder struct { type dnsseeder struct {
uptime time.Time uptime time.Time
theList map[string]*twistee theList map[string]*twistee
@ -144,30 +133,16 @@ func (s *dnsseeder) startCrawlers() {
continue continue
} }
// all looks go so start a go routine to crawl the remote twistee // all looks good so start a go routine to crawl the remote twistee
go crawlTwistee(tw) go crawlTwistee(tw)
c.started++ c.started++
} }
log.Printf("stats - started crawler: %s total: %v started: %v\n", c.desc, c.totalCount, c.started) log.Printf("stats - started crawler: %s total: %v started: %v\n", c.desc, c.totalCount, c.started)
counts.mtx.Lock() // update the global stats in another goroutine to free the main goroutine
switch c.status { // for other work
case statusRG: go updateTwCounts(c.status, c.totalCount, c.started)
counts.RG = c.totalCount
counts.RGS = c.started
case statusCG:
counts.CG = c.totalCount
counts.CGS = c.started
case statusWG:
counts.WG = c.totalCount
counts.WGS = c.started
case statusNG:
counts.NG = c.totalCount
counts.NGS = c.started
}
counts.Total = tcount
counts.mtx.Unlock()
} }
log.Printf("stats - crawlers started. total twistees: %d\n", tcount) log.Printf("stats - crawlers started. total twistees: %d\n", tcount)

Write Preview
Loading…
Cancel
Save