|
|
@ -160,13 +160,16 @@ extern "C" void* ThreadCrawler(void* data) { |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
extern "C" int GetIPList(void *thread, addr_t *addr, int max, int ipv4, int ipv6); |
|
|
|
extern "C" int GetIPList(void *thread, addr_t *addr, int max, int ipv4, int ipv6); |
|
|
|
|
|
|
|
extern "C" int GetIPListNonStd(void *thread, addr_t *addr, int max, int ipv4, int ipv6); |
|
|
|
|
|
|
|
|
|
|
|
class CDnsThread { |
|
|
|
class CDnsThread { |
|
|
|
public: |
|
|
|
public: |
|
|
|
dns_opt_t dns_opt; // must be first
|
|
|
|
dns_opt_t dns_opt; // must be first
|
|
|
|
const int id; |
|
|
|
const int id; |
|
|
|
vector<addr_t> cache; |
|
|
|
vector<addr_t> cache; |
|
|
|
|
|
|
|
vector<addr_t> cacheNonStd; // [MF] non std port
|
|
|
|
int nIPv4, nIPv6; |
|
|
|
int nIPv4, nIPv6; |
|
|
|
|
|
|
|
int nonStdIPv4, nonStdIPv6; |
|
|
|
time_t cacheTime; |
|
|
|
time_t cacheTime; |
|
|
|
unsigned int cacheHits; |
|
|
|
unsigned int cacheHits; |
|
|
|
uint64_t dbQueries; |
|
|
|
uint64_t dbQueries; |
|
|
@ -180,29 +183,43 @@ public: |
|
|
|
time_t now = time(NULL); |
|
|
|
time_t now = time(NULL); |
|
|
|
cacheHits++; |
|
|
|
cacheHits++; |
|
|
|
if (force || cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) { |
|
|
|
if (force || cacheHits > (cache.size()*cache.size()/400) || (cacheHits*cacheHits > cache.size() / 20 && (now - cacheTime > 5))) { |
|
|
|
set<CNetAddr> ips; |
|
|
|
set<CService> ips; |
|
|
|
db.GetIPs(ips, 1000, nets); |
|
|
|
db.GetIPs(ips, 1000, nets); |
|
|
|
dbQueries++; |
|
|
|
dbQueries++; |
|
|
|
cache.clear(); |
|
|
|
cache.clear(); |
|
|
|
nIPv4 = 0; |
|
|
|
cacheNonStd.clear(); |
|
|
|
nIPv6 = 0; |
|
|
|
nIPv4 = nonStdIPv4 = 0; |
|
|
|
|
|
|
|
nIPv6 = nonStdIPv6 = 0; |
|
|
|
cache.reserve(ips.size()); |
|
|
|
cache.reserve(ips.size()); |
|
|
|
for (set<CNetAddr>::iterator it = ips.begin(); it != ips.end(); it++) { |
|
|
|
cacheNonStd.reserve(ips.size()); |
|
|
|
|
|
|
|
for (set<CService>::iterator it = ips.begin(); it != ips.end(); it++) { |
|
|
|
struct in_addr addr; |
|
|
|
struct in_addr addr; |
|
|
|
struct in6_addr addr6; |
|
|
|
struct in6_addr addr6; |
|
|
|
if ((*it).GetInAddr(&addr)) { |
|
|
|
if ((*it).GetInAddr(&addr)) { |
|
|
|
addr_t a; |
|
|
|
addr_t a; |
|
|
|
a.v = 4; |
|
|
|
a.v = 4; |
|
|
|
|
|
|
|
a.port = (*it).GetPort(); |
|
|
|
memcpy(&a.data.v4, &addr, 4); |
|
|
|
memcpy(&a.data.v4, &addr, 4); |
|
|
|
cache.push_back(a); |
|
|
|
if( a.port == GetDefaultPort() ) { |
|
|
|
nIPv4++; |
|
|
|
cache.push_back(a); |
|
|
|
|
|
|
|
nIPv4++; |
|
|
|
|
|
|
|
} else { |
|
|
|
|
|
|
|
cacheNonStd.push_back(a); |
|
|
|
|
|
|
|
nonStdIPv4++; |
|
|
|
|
|
|
|
} |
|
|
|
#ifdef USE_IPV6 |
|
|
|
#ifdef USE_IPV6 |
|
|
|
} else if ((*it).GetIn6Addr(&addr6)) { |
|
|
|
} else if ((*it).GetIn6Addr(&addr6)) { |
|
|
|
addr_t a; |
|
|
|
addr_t a; |
|
|
|
a.v = 6; |
|
|
|
a.v = 6; |
|
|
|
|
|
|
|
a.port = (*it).GetPort(); |
|
|
|
memcpy(&a.data.v6, &addr6, 16); |
|
|
|
memcpy(&a.data.v6, &addr6, 16); |
|
|
|
cache.push_back(a); |
|
|
|
if( a.port == GetDefaultPort() ) { |
|
|
|
nIPv6++; |
|
|
|
cache.push_back(a); |
|
|
|
|
|
|
|
nIPv6++; |
|
|
|
|
|
|
|
} else { |
|
|
|
|
|
|
|
cacheNonStd.push_back(a); |
|
|
|
|
|
|
|
nonStdIPv6++; |
|
|
|
|
|
|
|
} |
|
|
|
#endif |
|
|
|
#endif |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
|
} |
|
|
@ -218,6 +235,7 @@ public: |
|
|
|
dns_opt.datattl = 60; |
|
|
|
dns_opt.datattl = 60; |
|
|
|
dns_opt.nsttl = 40000; |
|
|
|
dns_opt.nsttl = 40000; |
|
|
|
dns_opt.cb = GetIPList; |
|
|
|
dns_opt.cb = GetIPList; |
|
|
|
|
|
|
|
dns_opt.cbNonStd = GetIPListNonStd; |
|
|
|
dns_opt.port = opts->nPort; |
|
|
|
dns_opt.port = opts->nPort; |
|
|
|
dns_opt.nRequests = 0; |
|
|
|
dns_opt.nRequests = 0; |
|
|
|
cache.clear(); |
|
|
|
cache.clear(); |
|
|
@ -235,6 +253,18 @@ public: |
|
|
|
} |
|
|
|
} |
|
|
|
}; |
|
|
|
}; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
unsigned short crc16(unsigned char* data_p, unsigned char length){ |
|
|
|
|
|
|
|
unsigned char x; |
|
|
|
|
|
|
|
unsigned short crc = 0xFFFF; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
while (length--){ |
|
|
|
|
|
|
|
x = crc >> 8 ^ *data_p++; |
|
|
|
|
|
|
|
x ^= x>>4; |
|
|
|
|
|
|
|
crc = (crc << 8) ^ ((unsigned short)(x << 12)) ^ ((unsigned short)(x <<5)) ^ ((unsigned short)x); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return crc; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6) { |
|
|
|
extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6) { |
|
|
|
CDnsThread *thread = (CDnsThread*)data; |
|
|
|
CDnsThread *thread = (CDnsThread*)data; |
|
|
|
thread->cacheHit(); |
|
|
|
thread->cacheHit(); |
|
|
@ -264,6 +294,48 @@ extern "C" int GetIPList(void *data, addr_t* addr, int max, int ipv4, int ipv6) |
|
|
|
return max; |
|
|
|
return max; |
|
|
|
} |
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
extern "C" int GetIPListNonStd(void *data, addr_t* addr, int max, int ipv4, int ipv6) { |
|
|
|
|
|
|
|
CDnsThread *thread = (CDnsThread*)data; |
|
|
|
|
|
|
|
thread->cacheHit(); |
|
|
|
|
|
|
|
unsigned int size = thread->cacheNonStd.size(); |
|
|
|
|
|
|
|
unsigned int maxmax = (ipv4 ? thread->nonStdIPv4 : 0) + (ipv6 ? thread->nonStdIPv6 : 0); |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if (max > 2 * size) |
|
|
|
|
|
|
|
max = 2 * size; |
|
|
|
|
|
|
|
if (max > 2 * maxmax) |
|
|
|
|
|
|
|
max = 2 * maxmax; |
|
|
|
|
|
|
|
max -= max % 2; |
|
|
|
|
|
|
|
int i=0; |
|
|
|
|
|
|
|
while (i+1<max) { |
|
|
|
|
|
|
|
int j = i/2 + (rand() % (size - i/2)); |
|
|
|
|
|
|
|
unsigned short crcAddr = 0; |
|
|
|
|
|
|
|
do { |
|
|
|
|
|
|
|
bool ok = (ipv4 && thread->cacheNonStd[j].v == 4) || |
|
|
|
|
|
|
|
(ipv6 && thread->cacheNonStd[j].v == 6); |
|
|
|
|
|
|
|
if (ok) break; |
|
|
|
|
|
|
|
j++; |
|
|
|
|
|
|
|
if (j==size) |
|
|
|
|
|
|
|
j=i/2; |
|
|
|
|
|
|
|
} while(1); |
|
|
|
|
|
|
|
addr[i] = thread->cacheNonStd[j]; |
|
|
|
|
|
|
|
thread->cacheNonStd[j] = thread->cacheNonStd[i/2]; |
|
|
|
|
|
|
|
thread->cacheNonStd[i/2] = addr[i]; |
|
|
|
|
|
|
|
|
|
|
|
|
|
|
|
if( addr[i].v == 4 ) { |
|
|
|
|
|
|
|
crcAddr = crc16(addr[i].data.v4,4); |
|
|
|
|
|
|
|
} else { |
|
|
|
|
|
|
|
crcAddr = crc16(addr[i].data.v6,16); |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
addr[i+1].v = 4; |
|
|
|
|
|
|
|
addr[i+1].data.v4[0] = crcAddr >> 8; |
|
|
|
|
|
|
|
addr[i+1].data.v4[1] = crcAddr & 0xff; |
|
|
|
|
|
|
|
addr[i+1].data.v4[2] = addr[i].port >> 8; |
|
|
|
|
|
|
|
addr[i+1].data.v4[3] = addr[i].port & 0xff; |
|
|
|
|
|
|
|
i+=2; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
return max; |
|
|
|
|
|
|
|
} |
|
|
|
|
|
|
|
|
|
|
|
vector<CDnsThread*> dnsThread; |
|
|
|
vector<CDnsThread*> dnsThread; |
|
|
|
|
|
|
|
|
|
|
|
extern "C" void* ThreadDNS(void* arg) { |
|
|
|
extern "C" void* ThreadDNS(void* arg) { |
|
|
|