twisterarmy
/
dnsseeder
mirror of https://github.com/twisterarmy/dnsseeder.git
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Improve global stats processing

master
Lyndsay Roger 9 years ago
parent
6a18e1aaaf
commit
b4cae52c0f
4 changed files with 107 additions and 51 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Split View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 18
      dns.go
  2. 53
      http.go
  3. 32
      main.go
  4. 55
      seeder.go

18
dns.go
Unescape View File

@ -141,21 +141,24 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) { @@ -141,21 +141,24 @@ func handleDNSStd(w dns.ResponseWriter, r *dns.Msg) {
m.Answer = getv4stdRR()
latest.mtx.RUnlock()
qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Std)
case dns.TypeAAAA:
latest.mtx.RLock()
m.Answer = getv6stdRR()
latest.mtx.RUnlock()
qtype = "AAAA"
go updateDNSCounts(dnsV6Std)
default:
// return no answer to all other queries
}
w.WriteMsg(m)
if config.verbose {
log.Printf("status - DNS response Type: standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
}
// FIXME - add stats and query counts
w.WriteMsg(m)
}
// handleDNSNon processes a DNS request from remote client and returns
@ -177,21 +180,24 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) { @@ -177,21 +180,24 @@ func handleDNSNon(w dns.ResponseWriter, r *dns.Msg) {
m.Answer = getv4nonRR()
latest.mtx.RUnlock()
qtype = "A"
// start a goroutine to update the global counters then get back to answering this request
go updateDNSCounts(dnsV4Non)
case dns.TypeAAAA:
latest.mtx.RLock()
m.Answer = getv6nonRR()
latest.mtx.RUnlock()
qtype = "AAAA"
go updateDNSCounts(dnsV6Non)
default:
// return no answer to all other queries
}
w.WriteMsg(m)
if config.verbose {
log.Printf("status - DNS response Type: non-standard To IP: %s Query Type: %s\n", w.RemoteAddr().String(), qtype)
}
// FIXME - add stats and query counts
w.WriteMsg(m)
}
// serve starts the requested DNS server listening on the requested port

53
http.go
Unescape View File

@ -390,6 +390,43 @@ func generateWebStatus(status uint32) (ws []webstatus) { @@ -390,6 +390,43 @@ func generateWebStatus(status uint32) (ws []webstatus) {
// genHeader will output the standard header
func writeHeader(w http.ResponseWriter, r *http.Request) {
var hc struct {
RG uint32
RGS uint32
CG uint32
CGS uint32
WG uint32
WGS uint32
NG uint32
NGS uint32
Total uint32
V4Std uint32
V4Non uint32
V6Std uint32
V6Non uint32
DNSTotal uint32
}
// fill the structs so they can be displayed via the template
counts.mtx.RLock()
hc.RG = counts.TwStatus[statusRG]
hc.RGS = counts.TwStarts[statusRG]
hc.CG = counts.TwStatus[statusCG]
hc.CGS = counts.TwStarts[statusCG]
hc.WG = counts.TwStatus[statusWG]
hc.WGS = counts.TwStarts[statusWG]
hc.NG = counts.TwStatus[statusNG]
hc.NGS = counts.TwStarts[statusNG]
hc.Total = hc.RG + hc.CG + hc.WG + hc.NG
hc.V4Std = counts.DNSCounts[dnsV4Std]
hc.V4Non = counts.DNSCounts[dnsV4Non]
hc.V6Std = counts.DNSCounts[dnsV6Std]
hc.V6Non = counts.DNSCounts[dnsV6Non]
hc.DNSTotal = hc.V4Std + hc.V4Non + hc.V6Std + hc.V6Non
counts.mtx.RUnlock()
// we are using basic and simple html here. No fancy graphics or css
h := `
<!DOCTYPE HTML PUBLIC "-//W3C//DTD HTML 4.01 Transitional//EN" "http://www.w3.org/TR/html4/loose.dtd">
@ -401,22 +438,28 @@ func writeHeader(w http.ResponseWriter, r *http.Request) { @@ -401,22 +438,28 @@ func writeHeader(w http.ResponseWriter, r *http.Request) {
<a href="/statusNG">statusNG</a>
<a href="/dns">DNS</a>
<br>
Current Stats (count/started)
<table><tr><td>
Twistee Stats (count/started)<br>
<table border=1><tr>
<td>RG: {{.RG}}/{{.RGS}}</td><td>CG: {{.CG}}/{{.CGS}}</td><td>WG: {{.WG}}/{{.WGS}}</td><td>NG: {{.NG}}/{{.NGS}}</td><td>Total: {{.Total}}</td>
<td>RG: {{.RG}}/{{.RGS}}</td><td>CG: {{.CG}}/{{.CGS}}</td><td>WG: {{.WG}}/{{.WGS}}</td><td>NG: {{.NG}}/{{.NGS}}</td><td>Total: {{.Total}}</td>
</tr></table>
</td><td>
DNS Requests<br>
<table border=1><tr>
<td>V4 Std: {{.V4Std}}</td><td>V4 Non: {{.V4Non}}</td><td>V6 Std: {{.V6Std}}</td><td>V6 Non: {{.V6Non}}</td><td>Total: {{.DNSTotal}}</td>
</tr></table>
</td></tr></table>
</center>
<hr>
`
t := template.New("Header template")
t, err := t.Parse(h)
if err != nil {
log.Printf("error parsing template %v\n", err)
}
counts.mtx.RLock()
err = t.Execute(w, counts)
counts.mtx.RUnlock()
err = t.Execute(w, hc)
if err != nil {
log.Printf("error executing template %v\n", err)
}

32
main.go
Unescape View File

@ -9,12 +9,22 @@ import ( @@ -9,12 +9,22 @@ import (
"log"
"os"
"os/signal"
"sync"
"syscall"
"time"
"github.com/miekg/dns"
)
// twCounts holds various statistics about the running system
type twCounts struct {
TwStatus []uint32
TwStarts []uint32
DNSCounts []uint32
mtx sync.RWMutex
}
// configData holds information on the application
type configData struct {
host string
port string
@ -34,6 +44,11 @@ func main() { @@ -34,6 +44,11 @@ func main() {
// FIXME - update with git hash during build
config.version = "0.5.0"
// initialize the stats counters
counts.TwStatus = make([]uint32, maxStatusTypes)
counts.TwStarts = make([]uint32, maxStatusTypes)
counts.DNSCounts = make([]uint32, maxDNSTypes)
flag.StringVar(&config.host, "h", "", "DNS host to serve")
flag.StringVar(&config.port, "p", "8053", "Port to listen on")
flag.BoolVar(&config.verbose, "v", false, "Display verbose output")
@ -117,6 +132,23 @@ func main() { @@ -117,6 +132,23 @@ func main() {
fmt.Printf("\nProgram exiting. Bye\n")
}
// updateTwCounts runs in a goroutine and updates the global stats with the lates
// counts from a startCrawlers run
func updateTwCounts(status, total, started uint32) {
// update the stats counters
counts.mtx.Lock()
counts.TwStatus[status] = total
counts.TwStarts[status] = started
counts.mtx.Unlock()
}
// updateDNSCounts runs in a goroutine and updates the global stats for the number of DNS requests
func updateDNSCounts(dnsType uint32) {
counts.mtx.Lock()
counts.DNSCounts[dnsType]++
counts.mtx.Unlock()
}
/*
*/

55
seeder.go
Unescape View File

@ -27,32 +27,21 @@ const ( @@ -27,32 +27,21 @@ const (
maxTo = 250 // max seconds (4min 10 sec) for all comms to twistee to complete before we timeout
dnsV4Std = 1
dnsV4Non = 2
dnsV6Std = 3
dnsV6Non = 4
dnsInvalid = 0
dnsV4Std = 1
dnsV4Non = 2
dnsV6Std = 3
dnsV6Non = 4
maxDNSTypes = 5
// twistee status
statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected
statusCG = 2 // confirmed good. We have connected to the twistee and received addresses
statusWG = 3 // was good. Twistee was confirmed good but now having problems
statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
statusRG = 1 // reported good status. A remote twistee has reported this ip but we have not connected
statusCG = 2 // confirmed good. We have connected to the twistee and received addresses
statusWG = 3 // was good. Twistee was confirmed good but now having problems
statusNG = 4 // no good. Will be removed from theList after 24 hours to redure bouncing ip addresses
maxStatusTypes = 5
)
type twCounts struct {
RG uint32
RGS uint32
CG uint32
CGS uint32
WG uint32
WGS uint32
NG uint32
NGS uint32
Total int
mtx sync.RWMutex
}
type dnsseeder struct {
uptime time.Time
theList map[string]*twistee
@ -144,30 +133,16 @@ func (s *dnsseeder) startCrawlers() { @@ -144,30 +133,16 @@ func (s *dnsseeder) startCrawlers() {
continue
}
// all looks go so start a go routine to crawl the remote twistee
// all looks good so start a go routine to crawl the remote twistee
go crawlTwistee(tw)
c.started++
}
log.Printf("stats - started crawler: %s total: %v started: %v\n", c.desc, c.totalCount, c.started)
counts.mtx.Lock()
switch c.status {
case statusRG:
counts.RG = c.totalCount
counts.RGS = c.started
case statusCG:
counts.CG = c.totalCount
counts.CGS = c.started
case statusWG:
counts.WG = c.totalCount
counts.WGS = c.started
case statusNG:
counts.NG = c.totalCount
counts.NGS = c.started
}
counts.Total = tcount
counts.mtx.Unlock()
// update the global stats in another goroutine to free the main goroutine
// for other work
go updateTwCounts(c.status, c.totalCount, c.started)
}
log.Printf("stats - crawlers started. total twistees: %d\n", tcount)

Write Preview
Loading…
Cancel
Save