d4708
/
opentracker
mirror of git://erdgeist.org/opentracker
1
0
Fork 0
Code Issues Projects Releases Wiki Activity
Browse Source

Major rewrite of live bad network logging. You can now limit logging to any netmask. HTTP interface for that coming soon.

dynamic-accesslists
erdgeist 16 years ago
parent
d113912101
commit
6c19143bc1
1 changed files with 155 additions and 142 deletions
Whitespace
Show all changes Ignore whitespace when comparing lines Ignore changes in amount of whitespace Ignore changes in whitespace at EOL
Unified View
Diff Options
Show Stats Download Patch File Download Diff File
  1. 297
      ot_stats.c

297
ot_stats.c
Unescape View File

@ -17,6 +17,7 @@
/* Libowfat */ /* Libowfat */
#include "byte.h" #include "byte.h"
#include "io.h" #include "io.h"
#include "ip4.h"
#include "ip6.h" #include "ip6.h"
/* Opentracker */ /* Opentracker */
@ -56,28 +57,36 @@ static unsigned long long ot_overall_stall_count;
static time_t ot_start_time; static time_t ot_start_time;
#ifdef WANT_LOG_NETWORKS #define STATS_NETWORK_NODE_BITWIDTH 4
#define STATS_NETWORK_NODE_BITWIDTH 8 #define STATS_NETWORK_NODE_COUNT (1<<STATS_NETWORK_NODE_BITWIDTH)
#define STATS_NETWORK_NODE_COUNT (1<<STATS_NETWORK_NODE_BITWIDTH)
#define __BYTE(P,D) (((uint8_t*)P)[D/8])
#define __MSK (STATS_NETWORK_NODE_COUNT-1)
#define __SHFT(D) ((D^STATS_NETWORK_NODE_BITWIDTH)&STATS_NETWORK_NODE_BITWIDTH)
#define __LDR(P,D) ((__BYTE((P),(D))>>__SHFT((D)))&__MSK)
#define __STR(P,D,V) __BYTE((P),(D))=(__BYTE((P),(D))&~(__MSK<<__SHFT((D))))|((V)<<__SHFT((D)))
#ifdef WANT_V6 #ifdef WANT_V6
#define STATS_NETWORK_NODE_MAXDEPTH (48/8-1) #define STATS_NETWORK_NODE_MAXDEPTH (68-STATS_NETWORK_NODE_BITWIDTH)
#define STATS_NETWORK_NODE_LIMIT (48-STATS_NETWORK_NODE_BITWIDTH)
#else #else
#define STATS_NETWORK_NODE_MAXDEPTH (12+24/8-1) #define STATS_NETWORK_NODE_MAXDEPTH (28-STATS_NETWORK_NODE_BITWIDTH)
#define STATS_NETWORK_NODE_LIMIT (24-STATS_NETWORK_NODE_BITWIDTH)
#endif #endif
typedef union stats_network_node stats_network_node; typedef union stats_network_node stats_network_node;
union stats_network_node { union stats_network_node {
int counters[STATS_NETWORK_NODE_COUNT]; int counters[STATS_NETWORK_NODE_COUNT];
stats_network_node *children[STATS_NETWORK_NODE_COUNT]; stats_network_node *children[STATS_NETWORK_NODE_COUNT];
}; };
#ifdef WANT_LOG_NETWORKS
static stats_network_node *stats_network_counters_root = NULL; static stats_network_node *stats_network_counters_root = NULL;
#endif
static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) { static int stat_increase_network_count( stats_network_node **node, int depth, uintptr_t ip ) {
uint8_t *_ip = (uint8_t*)ip; int foo = __LDR(ip,depth);
int foo = _ip[depth];
if( !*node ) { if( !*node ) {
*node = malloc( sizeof( stats_network_node ) ); *node = malloc( sizeof( stats_network_node ) );
@ -87,7 +96,7 @@ static int stat_increase_network_count( stats_network_node **node, int depth, ui
} }
if( depth < STATS_NETWORK_NODE_MAXDEPTH ) if( depth < STATS_NETWORK_NODE_MAXDEPTH )
return stat_increase_network_count( &(*node)->children[ foo ], depth+1, ip ); return stat_increase_network_count( &(*node)->children[ foo ], depth+STATS_NETWORK_NODE_BITWIDTH, ip );
(*node)->counters[ foo ]++; (*node)->counters[ foo ]++;
return 0; return 0;
@ -97,11 +106,12 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth,
int i, rest = 0; int i, rest = 0;
if( !*node ) return 0; if( !*node ) return 0;
if( ++depth == STATS_NETWORK_NODE_MAXDEPTH ) depth += STATS_NETWORK_NODE_BITWIDTH;
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) { if( depth == STATS_NETWORK_NODE_MAXDEPTH ) {
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
rest += ((*node)->counters[i]>>=shift); rest += ((*node)->counters[i]>>=shift);
return rest; return rest;
} }
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) { for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
stats_network_node **childnode = &(*node)->children[i]; stats_network_node **childnode = &(*node)->children[i];
@ -120,56 +130,155 @@ static int stats_shift_down_network_count( stats_network_node **node, int depth,
return rest; return rest;
} }
static void stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, int *scores, ot_ip6 *networks, int network_count ) { static size_t stats_get_highscore_networks( stats_network_node *node, int depth, ot_ip6 node_value, size_t *scores, ot_ip6 *networks, int network_count, int limit ) {
uint8_t *_node_value = (uint8_t*)node_value; size_t score = 0;
int i; int i;
if( !node ) return; if( !node ) return 0;
if( depth < STATS_NETWORK_NODE_MAXDEPTH ) { if( depth < limit ) {
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
if( node->children[i] ) { if( node->children[i] ) {
_node_value[depth] = i; __STR(node_value,depth,i);
stats_get_highscore_networks( node->children[i], depth+1, node_value, scores, networks, network_count ); score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
}
} else
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
int j=1;
if( node->counters[i] <= scores[0] ) continue;
_node_value[depth] = i;
while( (j<network_count) && (node->counters[i]>scores[j] ) ) ++j;
--j;
memcpy( scores, scores + 1, j * sizeof( *scores ) );
memcpy( networks, networks + 1, j * sizeof( *networks ) );
scores[ j ] = node->counters[ i ];
memcpy( networks + j, _node_value, sizeof( *networks ) );
} }
return score;
}
if( depth > limit && depth < STATS_NETWORK_NODE_MAXDEPTH ) {
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
if( node->children[i] )
score += stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
return score;
}
if( depth > limit && depth == STATS_NETWORK_NODE_MAXDEPTH ) {
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i )
score += node->counters[i];
return score;
}
/* if( depth == limit ) */
for( i=0; i<STATS_NETWORK_NODE_COUNT; ++i ) {
int j=1;
size_t node_score;
if( depth == STATS_NETWORK_NODE_MAXDEPTH )
node_score = node->counters[i];
else
node_score = stats_get_highscore_networks( node->children[i], depth+STATS_NETWORK_NODE_BITWIDTH, node_value, scores, networks, network_count, limit );
score += node_score;
if( node_score <= scores[0] ) continue;
__STR(node_value,depth,i);
while( j < network_count && node_score > scores[j] ) ++j;
--j;
memcpy( scores, scores + 1, j * sizeof( *scores ) );
memcpy( networks, networks + 1, j * sizeof( *networks ) );
scores[ j ] = node_score;
memcpy( networks + j, node_value, sizeof( *networks ) );
}
return score;
} }
static size_t stats_return_busy_networks( char * reply ) { static size_t stats_return_busy_networks( char * reply, stats_network_node *tree, int amount ) {
ot_ip6 networks[256]; ot_ip6 networks[amount];
ot_ip6 node_value; ot_ip6 node_value;
int scores[256]; size_t scores[amount];
int i; int i;
char * r = reply; char * r = reply;
memset( scores, 0, sizeof( *scores ) * 256 ); memset( scores, 0, sizeof( scores ) );
memset( networks, 0, sizeof( *networks ) * 256 ); memset( networks, 0, sizeof( networks ) );
memset( node_value, 0, sizeof( node_value ) );
stats_get_highscore_networks( stats_network_counters_root, 0, node_value, scores, networks, 256 ); stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_MAXDEPTH );
for( i=255; i>=0; --i) { r += sprintf( r, "Networks, limit /%d:\n", STATS_NETWORK_NODE_MAXDEPTH+STATS_NETWORK_NODE_BITWIDTH );
r += sprintf( r, "%08i: ", scores[i] ); for( i=amount-1; i>=0; --i) {
r += fmt_ip6c( r, networks[i] ); if( scores[i] ) {
*r++ = '\n'; r += sprintf( r, "%08zd: ", scores[i] );
#ifdef WANT_V6
r += fmt_ip6c( r, networks[i] );
#else
r += fmt_ip4( r, networks[i]);
#endif
*r++ = '\n';
}
} }
memset( scores, 0, sizeof( scores ) );
memset( networks, 0, sizeof( networks ) );
memset( node_value, 0, sizeof( node_value ) );
stats_get_highscore_networks( tree, 0, node_value, scores, networks, amount, STATS_NETWORK_NODE_LIMIT );
r += sprintf( r, "\nNetworks, limit /%d:\n", STATS_NETWORK_NODE_LIMIT+STATS_NETWORK_NODE_BITWIDTH );
for( i=amount-1; i>=0; --i) {
if( scores[i] ) {
r += sprintf( r, "%08zd: ", scores[i] );
#ifdef WANT_V6
r += fmt_ip6c( r, networks[i] );
#else
r += fmt_ip4( r, networks[i] );
#endif
*r++ = '\n';
}
}
return r - reply; return r - reply;
} }
#endif static size_t stats_slash24s_txt( char *reply, size_t amount ) {
stats_network_node *slash24s_network_counters_root = NULL;
char *r=reply;
int bucket;
size_t i;
for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
for( i=0; i<torrents_list->size; ++i ) {
ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[i] ).peer_list;
ot_vector *bucket_list = &peer_list->peers;
int num_buckets = 1;
if( OT_PEERLIST_HASBUCKETS( peer_list ) ) {
num_buckets = bucket_list->size;
bucket_list = (ot_vector *)bucket_list->data;
}
while( num_buckets-- ) {
ot_peer *peers = (ot_peer*)bucket_list->data;
size_t numpeers = bucket_list->size;
while( numpeers-- )
if( stat_increase_network_count( &slash24s_network_counters_root, 0, (uintptr_t)(peers++) ) )
goto bailout_unlock;
++bucket_list;
}
}
mutex_bucket_unlock( bucket, 0 );
if( !g_opentracker_running )
goto bailout_error;
}
/* The tree is built. Now analyze */
r += stats_return_busy_networks( r, slash24s_network_counters_root, amount );
goto success;
bailout_unlock:
mutex_bucket_unlock( bucket, 0 );
bailout_error:
r = reply;
success:
stats_shift_down_network_count( &slash24s_network_counters_root, 0, STATS_NETWORK_NODE_MAXDEPTH*STATS_NETWORK_NODE_BITWIDTH );
if( slash24s_network_counters_root )
free( slash24s_network_counters_root );
return r-reply;
}
typedef struct { typedef struct {
unsigned long long torrent_count; unsigned long long torrent_count;
@ -234,98 +343,6 @@ size_t stats_top10_txt( char * reply ) {
return r - reply; return r - reply;
} }
/* This function collects 4096 /24s in 4096 possible
malloc blocks
*/
static size_t stats_slash24s_txt( char * reply, size_t amount, uint32_t thresh ) {
#define NUM_TOPBITS 12
#define NUM_LOWBITS (24-NUM_TOPBITS)
#define NUM_BUFS (1<<NUM_TOPBITS)
#define NUM_S24S (1<<NUM_LOWBITS)
#define MSK_S24S (NUM_S24S-1)
uint32_t *counts[ NUM_BUFS ];
uint32_t slash24s[amount*2]; /* first dword amount, second dword subnet */
size_t i, j, k, l;
char *r = reply;
byte_zero( counts, sizeof( counts ) );
byte_zero( slash24s, amount * 2 * sizeof(uint32_t) );
r += sprintf( r, "Stats for all /24s with more than %u announced torrents:\n\n", thresh );
#if 0
/* XXX: TOOD: Doesn't work yet with new peer storage model */
for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) {
ot_vector *torrents_list = mutex_bucket_lock( bucket );
for( j=0; j<torrents_list->size; ++j ) {
ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list;
for( k=0; k<OT_POOLS_COUNT; ++k ) {
ot_peer *peers = peer_list->peers[k].data;
size_t numpeers = peer_list->peers[k].size;
for( l=0; l<numpeers; ++l ) {
uint32_t s24 = ntohl(*(uint32_t*)(peers+l)) >> 8;
uint32_t *count = counts[ s24 >> NUM_LOWBITS ];
if( !count ) {
count = malloc( sizeof(uint32_t) * NUM_S24S );
if( !count ) {
mutex_bucket_unlock( bucket, 0 );
goto bailout_cleanup;
}
byte_zero( count, sizeof( uint32_t ) * NUM_S24S );
counts[ s24 >> NUM_LOWBITS ] = count;
}
count[ s24 & MSK_S24S ]++;
}
}
}
mutex_bucket_unlock( bucket, 0 );
if( !g_opentracker_running )
goto bailout_cleanup;
}
#endif
k = l = 0; /* Debug: count allocated bufs */
for( i=0; i < NUM_BUFS; ++i ) {
uint32_t *count = counts[i];
if( !counts[i] )
continue;
++k; /* Debug: count allocated bufs */
for( j=0; j < NUM_S24S; ++j ) {
if( count[j] > thresh ) {
/* This subnet seems to announce more torrents than the last in our list */
int insert_pos = amount - 1;
while( ( insert_pos >= 0 ) && ( count[j] > slash24s[ 2 * insert_pos ] ) )
--insert_pos;
++insert_pos;
memcpy( slash24s + 2 * ( insert_pos + 1 ), slash24s + 2 * ( insert_pos ), 2 * sizeof( uint32_t ) * ( amount - insert_pos - 1 ) );
slash24s[ 2 * insert_pos ] = count[j];
slash24s[ 2 * insert_pos + 1 ] = ( i << NUM_TOPBITS ) + j;
if( slash24s[ 2 * amount - 2 ] > thresh )
thresh = slash24s[ 2 * amount - 2 ];
}
if( count[j] ) ++l;
}
free( count );
}
r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l );
for( i=0; i < amount; ++i )
if( slash24s[ 2*i ] >= thresh ) {
uint32_t ip = slash24s[ 2*i +1 ];
r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) );
}
return r - reply;
for( i=0; i < NUM_BUFS; ++i )
free( counts[i] );
return 0;
}
static unsigned long events_per_time( unsigned long long events, time_t t ) { static unsigned long events_per_time( unsigned long long events, time_t t ) {
return events / ( (unsigned int)t ? (unsigned int)t : 1 ); return events / ( (unsigned int)t ? (unsigned int)t : 1 );
} }
@ -532,10 +549,6 @@ size_t return_stats_for_tracker( char *reply, int mode, int format ) {
return stats_return_renew_bucket( reply ); return stats_return_renew_bucket( reply );
case TASK_STATS_SYNCS: case TASK_STATS_SYNCS:
return stats_return_sync_mrtg( reply ); return stats_return_sync_mrtg( reply );
#ifdef WANT_LOG_NETWORKS
case TASK_STATS_BUSY_NETWORKS:
return stats_return_busy_networks( reply );
#endif
default: default:
return 0; return 0;
} }
@ -552,7 +565,7 @@ static void stats_make( int *iovec_entries, struct iovec **iovector, ot_tasktype
switch( mode & TASK_TASK_MASK ) { switch( mode & TASK_TASK_MASK ) {
case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break; case TASK_STATS_TORRENTS: r += stats_torrents_mrtg( r ); break;
case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break; case TASK_STATS_PEERS: r += stats_peers_mrtg( r ); break;
case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 25, 16 ); break; case TASK_STATS_SLASH24S: r += stats_slash24s_txt( r, 128 ); break;
case TASK_STATS_TOP10: r += stats_top10_txt( r ); break; case TASK_STATS_TOP10: r += stats_top10_txt( r ); break;
case TASK_STATS_EVERYTHING: r += stats_return_everything( r ); break; case TASK_STATS_EVERYTHING: r += stats_return_everything( r ); break;
default: default:

Write Preview
Loading…
Cancel
Save