From 8900cc0dd980cb08a0af957a1d0dd849bf3c2ac6 Mon Sep 17 00:00:00 2001 From: erdgeist <> Date: Tue, 6 Nov 2007 11:58:32 +0000 Subject: [PATCH] No one can get access to buckets now without locking them. Also split up the trackerlogic.c-monster in functional sub-units. HEADS UP: this code is untested and not considered stable. --- Makefile | 6 +- opentracker.c | 2 + ot_clean.c | 119 ++++++++++ ot_clean.h | 15 ++ ot_mutex.c | 31 ++- ot_mutex.h | 9 +- ot_stats.c | 201 +++++++++++++++++ ot_stats.h | 13 ++ ot_sync.c | 107 +++++++++ ot_sync.h | 14 ++ ot_vector.c | 110 +++++++++ ot_vector.h | 26 +++ trackerlogic.c | 600 +++++-------------------------------------------- trackerlogic.h | 48 ++-- 14 files changed, 723 insertions(+), 578 deletions(-) create mode 100644 ot_clean.c create mode 100644 ot_clean.h create mode 100644 ot_stats.c create mode 100644 ot_stats.h create mode 100644 ot_sync.c create mode 100644 ot_sync.h create mode 100644 ot_vector.c create mode 100644 ot_vector.h diff --git a/Makefile b/Makefile index 4dd8957..2360025 100644 --- a/Makefile +++ b/Makefile @@ -1,13 +1,13 @@ CC?=gcc -FEATURES=#-DWANT_CLOSED_TRACKER -DWANT_UTORRENT1600_WORKAROUND #-DWANT_IP_FROM_QUERY_STRING -D_DEBUG_HTTPERROR -DWANT_TRACKER_SYNC +FEATURES=-DWANT_TRACKER_SYNC #-DWANT_CLOSED_TRACKER -DWANT_UTORRENT1600_WORKAROUND #-DWANT_IP_FROM_QUERY_STRING -D_DEBUG_HTTPERROR -DWANT_TRACKER_SYNC OPTS_debug=-g -ggdb #-pg # -fprofile-arcs -ftest-coverage OPTS_production=-s -Os CFLAGS+=-I../libowfat -Wall -pipe -Wextra #-pedantic #-ansi LDFLAGS+=-L../libowfat/ -lowfat BINARY = opentracker -HEADERS=trackerlogic.h scan_urlencoded_query.h mutex.h -SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c mutex.c +HEADERS=trackerlogic.h scan_urlencoded_query.h ot_mutex.h ot_stats.h ot_sync.h ot_vector.h ot_clean.h +SOURCES=opentracker.c trackerlogic.c scan_urlencoded_query.c ot_mutex.c ot_stats.c ot_sync.c ot_vector.c ot_clean.c all: $(BINARY) $(BINARY).debug diff --git a/opentracker.c b/opentracker.c index 00fe65f..0345177 100644 --- a/opentracker.c +++ b/opentracker.c @@ -29,6 +29,8 @@ #include "trackerlogic.h" #include "scan_urlencoded_query.h" +#include "ot_stats.h" +#include "ot_sync.h" /* Globals */ static unsigned long long ot_overall_tcp_connections = 0; diff --git a/ot_clean.c b/ot_clean.c new file mode 100644 index 0000000..46b3e0c --- /dev/null +++ b/ot_clean.c @@ -0,0 +1,119 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Libowfat */ +#include "byte.h" + +/* Opentracker */ +#include "trackerlogic.h" +#include "ot_mutex.h" + +/* To remember, when we last cleaned up */ +static ot_time all_torrents_clean[OT_BUCKET_COUNT]; + +/* Clean a single torrent + return 1 if torrent timed out +*/ +int clean_single_torrent( ot_torrent *torrent ) { + ot_peerlist *peer_list = torrent->peer_list; + size_t peers_count = 0, seeds_count; + time_t timedout = (int)( NOW - peer_list->base ); + int i; +#ifdef WANT_TRACKER_SYNC + char *new_peers; +#endif + + /* Torrent has idled out */ + if( timedout > OT_TORRENT_TIMEOUT ) + return 1; + + /* Nothing to be cleaned here? Test if torrent is worth keeping */ + if( timedout > OT_POOLS_COUNT ) { + if( !peer_list->peer_count ) + return peer_list->down_count ? 0 : 1; + timedout = OT_POOLS_COUNT; + } + + /* Release vectors that have timed out */ + for( i = OT_POOLS_COUNT - timedout; i < OT_POOLS_COUNT; ++i ) + free( peer_list->peers[i].data); + + /* Shift vectors back by the amount of pools that were shifted out */ + memmove( peer_list->peers + timedout, peer_list->peers, sizeof( ot_vector ) * ( OT_POOLS_COUNT - timedout ) ); + byte_zero( peer_list->peers, sizeof( ot_vector ) * timedout ); + + /* Shift back seed counts as well */ + memmove( peer_list->seed_counts + timedout, peer_list->seed_counts, sizeof( size_t ) * ( OT_POOLS_COUNT - timedout ) ); + byte_zero( peer_list->seed_counts, sizeof( size_t ) * timedout ); + +#ifdef WANT_TRACKER_SYNC + /* Save the block modified within last OT_POOLS_TIMEOUT */ + if( peer_list->peers[1].size && + ( new_peers = realloc( peer_list->changeset.data, sizeof( ot_peer ) * peer_list->peers[1].size ) ) ) + { + memmove( new_peers, peer_list->peers[1].data, peer_list->peers[1].size ); + peer_list->changeset.data = new_peers; + peer_list->changeset.size = sizeof( ot_peer ) * peer_list->peers[1].size; + } else { + free( peer_list->changeset.data ); + + memset( &peer_list->changeset, 0, sizeof( ot_vector ) ); + } +#endif + + peers_count = seeds_count = 0; + for( i = 0; i < OT_POOLS_COUNT; ++i ) { + peers_count += peer_list->peers[i].size; + seeds_count += peer_list->seed_counts[i]; + } + peer_list->seed_count = seeds_count; + peer_list->peer_count = peers_count; + + if( peers_count ) + peer_list->base = NOW; + else { + /* When we got here, the last time that torrent + has been touched is OT_POOLS_COUNT units before */ + peer_list->base = NOW - OT_POOLS_COUNT; + } + return 0; +} + +/* Clean up all peers in current bucket, remove timedout pools and + torrents */ +void clean_all_torrents( void ) { + ot_vector *torrents_list; + size_t i; + static int bucket; + ot_time time_now = NOW; + + /* Search for an uncleaned bucked */ + while( ( all_torrents_clean[bucket] == time_now ) && ( ++bucket < OT_BUCKET_COUNT ) ); + if( bucket >= OT_BUCKET_COUNT ) { + bucket = 0; return; + } + + all_torrents_clean[bucket] = time_now; + + torrents_list = mutex_bucket_lock( bucket ); + for( i=0; isize; ++i ) { + ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; + if( clean_single_torrent( torrent ) ) { + vector_remove_torrent( torrents_list, torrent ); + --i; continue; + } + } + mutex_bucket_unlock( bucket ); +} + +void clean_init( void ) { + byte_zero( all_torrents_clean, sizeof( all_torrents_clean ) ); +} + +void clean_deinit( void ) { + byte_zero( all_torrents_clean, sizeof( all_torrents_clean ) ); +} \ No newline at end of file diff --git a/ot_clean.h b/ot_clean.h new file mode 100644 index 0000000..9c2fe11 --- /dev/null +++ b/ot_clean.h @@ -0,0 +1,15 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef __OT_CLEAN_H__ +#define __OT_CLEAN_H__ + +#include "trackerlogic.h" + +void clean_init( void ); +void clean_deinit( void ); + +void clean_all_torrents( void ); +int clean_single_torrent( ot_torrent *torrent ); + +#endif \ No newline at end of file diff --git a/ot_mutex.c b/ot_mutex.c index 5c14e45..bb82f46 100644 --- a/ot_mutex.c +++ b/ot_mutex.c @@ -1,11 +1,19 @@ /* This software was written by Dirk Engling It is considered beerware. Prost. Skol. Cheers or whatever. */ +/* System */ #include #include +/* Libowfat */ +#include "byte.h" + +/* Opentracker */ #include "trackerlogic.h" -#include "mutex.h" +#include "ot_mutex.h" + +/* Our global all torrents list */ +static ot_vector all_torrents[OT_BUCKET_COUNT]; static int bucket_locklist[ OT_MAX_THREADS ]; static int bucket_locklist_count = 0; @@ -51,12 +59,23 @@ static void bucket_remove( int bucket ) { --bucket_locklist_count; } -void mutex_bucket_lock( int bucket ) { +ot_vector *mutex_bucket_lock( int bucket ) { pthread_mutex_lock( &bucket_mutex ); while( bucket_check( bucket ) ) pthread_cond_wait( &bucket_being_unlocked, &bucket_mutex ); bucket_push( bucket ); pthread_mutex_unlock( &bucket_mutex ); + return all_torrents + bucket; +} + +ot_vector *mutex_bucket_lock_by_hash( ot_hash *hash ) { + unsigned char *local_hash = hash[0]; + int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 ); + + /* Can block */ + mutex_bucket_lock( bucket ); + + return all_torrents + bucket; } void mutex_bucket_unlock( int bucket ) { @@ -66,12 +85,20 @@ void mutex_bucket_unlock( int bucket ) { pthread_mutex_unlock( &bucket_mutex ); } +void mutex_bucket_unlock_by_hash( ot_hash *hash ) { + unsigned char *local_hash = hash[0]; + int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 ); + mutex_bucket_unlock( bucket ); +} + void mutex_init( ) { pthread_mutex_init(&bucket_mutex, NULL); pthread_cond_init (&bucket_being_unlocked, NULL); + byte_zero( all_torrents, sizeof( all_torrents ) ); } void mutex_deinit( ) { pthread_mutex_destroy(&bucket_mutex); pthread_cond_destroy(&bucket_being_unlocked); + byte_zero( all_torrents, sizeof( all_torrents ) ); } diff --git a/ot_mutex.h b/ot_mutex.h index 8d91ab3..2d30c69 100644 --- a/ot_mutex.h +++ b/ot_mutex.h @@ -1,13 +1,16 @@ /* This software was written by Dirk Engling It is considered beerware. Prost. Skol. Cheers or whatever. */ -#ifndef __MUTEX_H__ -#define __MUTEX_H__ +#ifndef __OT_MUTEX_H__ +#define __OT_MUTEX_H__ void mutex_init( ); void mutex_deinit( ); -void mutex_bucket_lock( int bucket ); +ot_vector *mutex_bucket_lock( int bucket ); +ot_vector *mutex_bucket_lock_by_hash( ot_hash *hash ); + void mutex_bucket_unlock( int bucket ); +void mutex_bucket_unlock_by_hash( ot_hash *hash ); #endif diff --git a/ot_stats.c b/ot_stats.c new file mode 100644 index 0000000..215d557 --- /dev/null +++ b/ot_stats.c @@ -0,0 +1,201 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include +#include + +/* Libowfat */ +#include "byte.h" + +/* Opentracker */ +#include "trackerlogic.h" +#include "ot_mutex.h" +#include "ot_stats.h" + +/* Converter function from memory to human readable hex strings */ +static char*to_hex(char*d,ot_byte*s){const char*m="0123456789ABCDEF";char*e=d+40;while(d>4];*d++=m[*s++&15];}*d=0;return d;} + +typedef struct { size_t val; ot_torrent * torrent; } ot_record; + +/* Fetches stats from tracker */ +size_t return_stats_for_tracker( char *reply, int mode ) { + size_t torrent_count = 0, peer_count = 0, seed_count = 0, j; + ot_record top5s[5], top5c[5]; + char *r = reply; + int bucket; + + byte_zero( top5s, sizeof( top5s ) ); + byte_zero( top5c, sizeof( top5c ) ); + + for( bucket=0; bucketsize; + for( j=0; jsize; ++j ) { + ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; + if( mode == STATS_TOP5 ) { + int idx = 4; while( (idx >= 0) && ( peer_list->peer_count > top5c[idx].val ) ) --idx; + if ( idx++ != 4 ) { + memmove( top5c + idx + 1, top5c + idx, ( 4 - idx ) * sizeof( ot_record ) ); + top5c[idx].val = peer_list->peer_count; + top5c[idx].torrent = (ot_torrent*)(torrents_list->data) + j; + } + idx = 4; while( (idx >= 0) && ( peer_list->seed_count > top5s[idx].val ) ) --idx; + if ( idx++ != 4 ) { + memmove( top5s + idx + 1, top5s + idx, ( 4 - idx ) * sizeof( ot_record ) ); + top5s[idx].val = peer_list->seed_count; + top5s[idx].torrent = (ot_torrent*)(torrents_list->data) + j; + } + } + peer_count += peer_list->peer_count; seed_count += peer_list->seed_count; + } + mutex_bucket_unlock( bucket ); + } + if( mode == STATS_TOP5 ) { + char hex_out[42]; + int idx; + r += sprintf( r, "Top5 torrents by peers:\n" ); + for( idx=0; idx<5; ++idx ) + if( top5c[idx].torrent ) + r += sprintf( r, "\t%zd\t%s\n", top5c[idx].val, to_hex( hex_out, top5c[idx].torrent->hash) ); + r += sprintf( r, "Top5 torrents by seeds:\n" ); + for( idx=0; idx<5; ++idx ) + if( top5s[idx].torrent ) + r += sprintf( r, "\t%zd\t%s\n", top5s[idx].val, to_hex( hex_out, top5s[idx].torrent->hash) ); + } else + r += sprintf( r, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", peer_count, seed_count, torrent_count ); + + return r - reply; +} + +/* This function collects 4096 /24s in 4096 possible + malloc blocks +*/ +size_t return_stats_for_slash24s( char *reply, size_t amount, ot_dword thresh ) { + +#define NUM_TOPBITS 12 +#define NUM_LOWBITS (24-NUM_TOPBITS) +#define NUM_BUFS (1<size; ++j ) { + ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; + for( k=0; kpeers[k].data; + size_t numpeers = peer_list->peers[k].size; + for( l=0; l> 8; + ot_dword *count = counts[ s24 >> NUM_LOWBITS ]; + if( !count ) { + count = malloc( sizeof(ot_dword) * NUM_S24S ); + if( !count ) + goto bailout_cleanup; + byte_zero( count, sizeof( ot_dword ) * NUM_S24S ); + counts[ s24 >> NUM_LOWBITS ] = count; + } + count[ s24 & MSK_S24S ]++; + } + } + } + mutex_bucket_unlock( bucket ); + } + + k = l = 0; /* Debug: count allocated bufs */ + for( i=0; i < NUM_BUFS; ++i ) { + ot_dword *count = counts[i]; + if( !counts[i] ) + continue; + ++k; /* Debug: count allocated bufs */ + for( j=0; j < NUM_S24S; ++j ) { + if( count[j] > thresh ) { + /* This subnet seems to announce more torrents than the last in our list */ + int insert_pos = amount - 1; + while( ( insert_pos >= 0 ) && ( count[j] > slash24s[ 2 * insert_pos ] ) ) + --insert_pos; + ++insert_pos; + memmove( slash24s + 2 * ( insert_pos + 1 ), slash24s + 2 * ( insert_pos ), 2 * sizeof( ot_dword ) * ( amount - insert_pos - 1 ) ); + slash24s[ 2 * insert_pos ] = count[j]; + slash24s[ 2 * insert_pos + 1 ] = ( i << NUM_TOPBITS ) + j; + if( slash24s[ 2 * amount - 2 ] > thresh ) + thresh = slash24s[ 2 * amount - 2 ]; + } + if( count[j] ) ++l; + } + free( count ); + } + + r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l ); + + for( i=0; i < amount; ++i ) + if( slash24s[ 2*i ] >= thresh ) { + ot_dword ip = slash24s[ 2*i +1 ]; + r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) ); + } + + return r - reply; + +bailout_cleanup: + + for( i=0; i < NUM_BUFS; ++i ) + free( counts[i] ); + + return 0; +} + +size_t return_memstat_for_tracker( char **reply ) { + size_t torrent_count = 0, j; + size_t allocated, replysize; + ot_vector *torrents_list; + int bucket, k; + char *r; + + for( bucket=0; bucketsize; + mutex_bucket_unlock(bucket); + } + + allocated = OT_BUCKET_COUNT*32 + (43+OT_POOLS_COUNT*32)*torrent_count; + if( !( r = *reply = mmap( NULL, allocated, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ) ) ) return 0; + + for( bucket=0; bucketsize, (unsigned int)torrents_list->space ); + mutex_bucket_unlock(bucket); + } + + for( bucket=0; bucketsize; ++j ) { + ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; + ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[j] ).hash; + r += sprintf( r, "\n%s:\n", to_hex( hex_out, (ot_byte*)hash) ); + for( k=0; kpeers[k].size), (unsigned int)peer_list->peers[k].space ); + } + mutex_bucket_unlock(bucket); + } + + replysize = ( r - *reply ); + fix_mmapallocation( *reply, allocated, replysize ); + + return replysize; +} \ No newline at end of file diff --git a/ot_stats.h b/ot_stats.h new file mode 100644 index 0000000..e4d7c06 --- /dev/null +++ b/ot_stats.h @@ -0,0 +1,13 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef __OT_STATS_H__ +#define __OT_STATS_H__ + +enum { STATS_CONNS, STATS_PEERS, STATS_TOP5, STATS_DMEM, STATS_TCP, STATS_UDP, STATS_SLASH24S, SYNC_IN, SYNC_OUT, STATS_FULLSCRAPE }; + +size_t return_stats_for_tracker( char *reply, int mode ); +size_t return_stats_for_slash24s( char *reply, size_t amount, ot_dword thresh ); +size_t return_memstat_for_tracker( char **reply ); + +#endif diff --git a/ot_sync.c b/ot_sync.c new file mode 100644 index 0000000..6e95a98 --- /dev/null +++ b/ot_sync.c @@ -0,0 +1,107 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include +#include +#include + +/* Libowfat */ +#include "scan.h" +#include "byte.h" + +/* Opentracker */ +#include "trackerlogic.h" +#include "ot_mutex.h" +#include "ot_sync.h" + +#ifdef WANT_TRACKER_SYNC +/* Import Changeset from an external authority + format: d4:syncd[..]ee + [..]: ( 20:01234567890abcdefghij16:XXXXYYYY )+ +*/ +int add_changeset_to_tracker( ot_byte *data, size_t len ) { + ot_hash *hash; + ot_byte *end = data + len; + unsigned long peer_count; + + /* We do know, that the string is \n terminated, so it cant + overflow */ + if( byte_diff( data, 8, "d4:syncd" ) ) return -1; + data += 8; + + while( 1 ) { + if( byte_diff( data, 3, "20:" ) ) { + if( byte_diff( data, 2, "ee" ) ) + return -1; + return 0; + } + data += 3; + hash = (ot_hash*)data; + data += sizeof( ot_hash ); + + /* Scan string length indicator */ + data += ( len = scan_ulong( (char*)data, &peer_count ) ); + + /* If no long was scanned, it is not divisible by 8, it is not + followed by a colon or claims to need to much memory, we fail */ + if( !len || !peer_count || ( peer_count & 7 ) || ( *data++ != ':' ) || ( data + peer_count > end ) ) + return -1; + + while( peer_count > 0 ) { + add_peer_to_torrent( hash, (ot_peer*)data, 1 ); + data += 8; peer_count -= 8; + } + } + return 0; +} + +/* Proposed output format + d4:syncd20:8*N:(xxxxyyyy)*Nee +*/ +size_t return_changeset_for_tracker( char **reply ) { + size_t allocated = 0, i, replysize; + ot_vector *torrents_list; + int bucket; + char *r; + + /* Maybe there is time to clean_all_torrents(); */ + + /* Determine space needed for whole changeset */ + for( bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket ) { + torrents_list = mutex_bucket_lock(bucket); + for( i=0; isize; ++i ) { + ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; + allocated += sizeof( ot_hash ) + sizeof(ot_peer) * torrent->peer_list->changeset.size + 13; + } + mutex_bucket_unlock(bucket); + } + + /* add "d4:syncd" and "ee" */ + allocated += 8 + 2; + + if( !( r = *reply = mmap( NULL, allocated, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ) ) ) + return 0; + + memmove( r, "d4:syncd", 8 ); r += 8; + for( bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket ) { + torrents_list = mutex_bucket_lock(bucket); + for( i=0; isize; ++i ) { + ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; + const size_t byte_count = sizeof(ot_peer) * torrent->peer_list->changeset.size; + *r++ = '2'; *r++ = '0'; *r++ = ':'; + memmove( r, torrent->hash, sizeof( ot_hash ) ); r += sizeof( ot_hash ); + r += sprintf( r, "%zd:", byte_count ); + memmove( r, torrent->peer_list->changeset.data, byte_count ); r += byte_count; + } + mutex_bucket_unlock(bucket); + } + *r++ = 'e'; *r++ = 'e'; + + replysize = ( r - *reply ); + fix_mmapallocation( *reply, allocated, replysize ); + + return replysize; +} +#endif diff --git a/ot_sync.h b/ot_sync.h new file mode 100644 index 0000000..cf0fb3d --- /dev/null +++ b/ot_sync.h @@ -0,0 +1,14 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef __OT_SYNC_H__ +#define __OT_SYNC_H__ + +#include "trackerlogic.h" + +#ifdef WANT_TRACKER_SYNC +size_t return_changeset_for_tracker( char **reply ); +int add_changeset_to_tracker( ot_byte *data, size_t len ); +#endif + +#endif \ No newline at end of file diff --git a/ot_vector.c b/ot_vector.c new file mode 100644 index 0000000..aa71279 --- /dev/null +++ b/ot_vector.c @@ -0,0 +1,110 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +/* System */ +#include +#include + +/* Opentracker */ +#include "trackerlogic.h" +#include "ot_vector.h" + +/* This function gives us a binary search that returns a pointer, even if + no exact match is found. In that case it sets exactmatch 0 and gives + calling functions the chance to insert data +*/ +void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size, + size_t compare_size, int *exactmatch ) { + size_t mc = member_count; + ot_byte *lookat = ((ot_byte*)base) + member_size * (member_count >> 1); + *exactmatch = 1; + + while( mc ) { + int cmp = memcmp( lookat, key, compare_size); + if (cmp == 0) return (void *)lookat; + if (cmp < 0) { + base = (void*)(lookat + member_size); + --mc; + } + mc >>= 1; + lookat = ((ot_byte*)base) + member_size * (mc >> 1); + } + *exactmatch = 0; + return (void*)lookat; +} + +/* This is the generic insert operation for our vector type. + It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with + those of objects in vector. Our special "binary_search" function does that and either returns the match or a + pointer to where the object is to be inserted. vector_find_or_insert makes space for the object and copies it, + if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert + took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector. +*/ +void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch ) { + ot_byte *match = binary_search( key, vector->data, vector->size, member_size, compare_size, exactmatch ); + + if( *exactmatch ) return match; + + if( vector->size + 1 >= vector->space ) { + size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; + ot_byte *new_data = realloc( vector->data, new_space * member_size ); + if( !new_data ) return NULL; + + /* Adjust pointer if it moved by realloc */ + match = new_data + (match - (ot_byte*)vector->data); + + vector->data = new_data; + vector->space = new_space; + } + memmove( match + member_size, match, ((ot_byte*)vector->data) + member_size * vector->size - match ); + vector->size++; + return match; +} + +/* This is the non-generic delete from vector-operation specialized for peers in pools. + Set hysteresis == 0 if you expect the vector not to ever grow again. + It returns 0 if no peer was found (and thus not removed) + 1 if a non-seeding peer was removed + 2 if a seeding peer was removed +*/ +int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis ) { + int exactmatch; + size_t shrink_thresh = hysteresis ? OT_VECTOR_SHRINK_THRESH : OT_VECTOR_SHRINK_RATIO; + ot_peer *end = ((ot_peer*)vector->data) + vector->size; + ot_peer *match; + + if( !vector->size ) return 0; + match = binary_search( peer, vector->data, vector->size, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch ); + + if( !exactmatch ) return 0; + exactmatch = ( OT_FLAG( match ) & PEER_FLAG_SEEDING ) ? 2 : 1; + memmove( match, match + 1, sizeof(ot_peer) * ( end - match - 1 ) ); + if( ( --vector->size * shrink_thresh < vector->space ) && ( vector->space > OT_VECTOR_MIN_MEMBERS ) ) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + vector->data = realloc( vector->data, vector->space * sizeof( ot_peer ) ); + } + if( !vector->size ) { + /* for peer pools its safe to let them go, + in 999 of 1000 this happens in older pools, that won't ever grow again */ + free( vector->data ); + vector->data = NULL; + vector->space = 0; + } + return exactmatch; +} + +void vector_remove_torrent( ot_vector *vector, ot_torrent *match ) { + ot_torrent *end = ((ot_torrent*)vector->data) + vector->size; + + if( !vector->size ) return; + + /* If this is being called after a unsuccessful malloc() for peer_list + in add_peer_to_torrent, match->peer_list actually might be NULL */ + if( match->peer_list) free_peerlist( match->peer_list ); + + memmove( match, match + 1, sizeof(ot_torrent) * ( end - match - 1 ) ); + if( ( --vector->size * OT_VECTOR_SHRINK_THRESH < vector->space ) && ( vector->space > OT_VECTOR_MIN_MEMBERS ) ) { + vector->space /= OT_VECTOR_SHRINK_RATIO; + vector->data = realloc( vector->data, vector->space * sizeof( ot_torrent ) ); + } +} diff --git a/ot_vector.h b/ot_vector.h new file mode 100644 index 0000000..8863855 --- /dev/null +++ b/ot_vector.h @@ -0,0 +1,26 @@ +/* This software was written by Dirk Engling + It is considered beerware. Prost. Skol. Cheers or whatever. */ + +#ifndef __OT_VECTOR_H__ +#define __OT_VECTOR_H__ + +#include "trackerlogic.h" + +#define OT_VECTOR_MIN_MEMBERS 4 +#define OT_VECTOR_GROW_RATIO 8 +#define OT_VECTOR_SHRINK_THRESH 6 +#define OT_VECTOR_SHRINK_RATIO 4 +typedef struct { + void *data; + size_t size; + size_t space; +} ot_vector; + +void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size, + size_t compare_size, int *exactmatch ); +void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch ); + +int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis ); +void vector_remove_torrent( ot_vector *vector, ot_torrent *match ); + +#endif \ No newline at end of file diff --git a/trackerlogic.c b/trackerlogic.c index 2725399..c09f1b9 100644 --- a/trackerlogic.c +++ b/trackerlogic.c @@ -1,149 +1,33 @@ /* This software was written by Dirk Engling It is considered beerware. Prost. Skol. Cheers or whatever. */ -#include "trackerlogic.h" - +/* System */ #include #include #include -#include #include #include -#include #include #include #include -#include -#include +/* Libowfat */ #include "scan.h" #include "byte.h" -#include "mutex.h" -/* GLOBAL VARIABLES */ +/* Opentracker */ +#include "trackerlogic.h" +#include "ot_mutex.h" +#include "ot_stats.h" +#include "ot_clean.h" -/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs - Sort key is, of course, its hash */ -#define OT_BUCKET_COUNT 1024 -static ot_vector all_torrents[OT_BUCKET_COUNT]; -static ot_time all_torrents_clean[OT_BUCKET_COUNT]; +/* GLOBAL VARIABLES */ #if defined ( WANT_BLACKLISTING ) || defined( WANT_CLOSED_TRACKER ) static ot_vector accesslist; #define WANT_ACCESS_CONTROL #endif -static int clean_single_torrent( ot_torrent *torrent ); - -/* these functions protect our buckets from other threads that - try to commit announces or clean up */ -static ot_vector *lock_bucket_by_hash( ot_hash *hash ) { - unsigned char *local_hash = hash[0]; - int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 ); - - /* Can block */ - mutex_bucket_lock( bucket ); - - return all_torrents + bucket; -} - -static void *unlock_bucket_by_hash( ot_hash *hash ) { - unsigned char *local_hash = hash[0]; - int bucket = ( local_hash[0] << 2 ) | ( local_hash[1] >> 6 ); - mutex_bucket_unlock( bucket ); - - /* To make caller's code look better, allow - return unlock_bucket_by_hash() */ - return NULL; -} - -/* Converter function from memory to human readable hex strings */ -static char*to_hex(char*d,ot_byte*s){const char*m="0123456789ABCDEF";char*e=d+40;while(d>4];*d++=m[*s++&15];}*d=0;return d;} - -/* This function gives us a binary search that returns a pointer, even if - no exact match is found. In that case it sets exactmatch 0 and gives - calling functions the chance to insert data -*/ -static void *binary_search( const void * const key, const void * base, const size_t member_count, const size_t member_size, - size_t compare_size, int *exactmatch ) { - size_t mc = member_count; - ot_byte *lookat = ((ot_byte*)base) + member_size * (member_count >> 1); - *exactmatch = 1; - - while( mc ) { - int cmp = memcmp( lookat, key, compare_size); - if (cmp == 0) return (void *)lookat; - if (cmp < 0) { - base = (void*)(lookat + member_size); - --mc; - } - mc >>= 1; - lookat = ((ot_byte*)base) + member_size * (mc >> 1); - } - *exactmatch = 0; - return (void*)lookat; -} - -/* This is the generic insert operation for our vector type. - It tries to locate the object at "key" with size "member_size" by comparing its first "compare_size" bytes with - those of objects in vector. Our special "binary_search" function does that and either returns the match or a - pointer to where the object is to be inserted. vector_find_or_insert makes space for the object and copies it, - if it wasn't found in vector. Caller needs to check the passed "exactmatch" variable to see, whether an insert - took place. If resizing the vector failed, NULL is returned, else the pointer to the object in vector. -*/ -static void *vector_find_or_insert( ot_vector *vector, void *key, size_t member_size, size_t compare_size, int *exactmatch ) { - ot_byte *match = binary_search( key, vector->data, vector->size, member_size, compare_size, exactmatch ); - - if( *exactmatch ) return match; - - if( vector->size + 1 >= vector->space ) { - size_t new_space = vector->space ? OT_VECTOR_GROW_RATIO * vector->space : OT_VECTOR_MIN_MEMBERS; - ot_byte *new_data = realloc( vector->data, new_space * member_size ); - if( !new_data ) return NULL; - - /* Adjust pointer if it moved by realloc */ - match = new_data + (match - (ot_byte*)vector->data); - - vector->data = new_data; - vector->space = new_space; - } - memmove( match + member_size, match, ((ot_byte*)vector->data) + member_size * vector->size - match ); - vector->size++; - return match; -} - -/* This is the non-generic delete from vector-operation specialized for peers in pools. - Set hysteresis == 0 if you expect the vector not to ever grow again. - It returns 0 if no peer was found (and thus not removed) - 1 if a non-seeding peer was removed - 2 if a seeding peer was removed -*/ -static int vector_remove_peer( ot_vector *vector, ot_peer *peer, int hysteresis ) { - int exactmatch; - size_t shrink_thresh = hysteresis ? OT_VECTOR_SHRINK_THRESH : OT_VECTOR_SHRINK_RATIO; - ot_peer *end = ((ot_peer*)vector->data) + vector->size; - ot_peer *match; - - if( !vector->size ) return 0; - match = binary_search( peer, vector->data, vector->size, sizeof( ot_peer ), OT_PEER_COMPARE_SIZE, &exactmatch ); - - if( !exactmatch ) return 0; - exactmatch = ( OT_FLAG( match ) & PEER_FLAG_SEEDING ) ? 2 : 1; - memmove( match, match + 1, sizeof(ot_peer) * ( end - match - 1 ) ); - if( ( --vector->size * shrink_thresh < vector->space ) && ( vector->space > OT_VECTOR_MIN_MEMBERS ) ) { - vector->space /= OT_VECTOR_SHRINK_RATIO; - vector->data = realloc( vector->data, vector->space * sizeof( ot_peer ) ); - } - if( !vector->size ) { - /* for peer pools its safe to let them go, - in 999 of 1000 this happens in older pools, that won't ever grow again */ - free( vector->data ); - vector->data = NULL; - vector->space = 0; - } - return exactmatch; -} - -static void free_peerlist( ot_peerlist *peer_list ) { +void free_peerlist( ot_peerlist *peer_list ) { size_t i; for( i=0; ipeers[i].data ) @@ -154,27 +38,11 @@ static void free_peerlist( ot_peerlist *peer_list ) { free( peer_list ); } -static void vector_remove_torrent( ot_vector *vector, ot_torrent *match ) { - ot_torrent *end = ((ot_torrent*)vector->data) + vector->size; - - if( !vector->size ) return; - - /* If this is being called after a unsuccessful malloc() for peer_list - in add_peer_to_torrent, match->peer_list actually might be NULL */ - if( match->peer_list) free_peerlist( match->peer_list ); - - memmove( match, match + 1, sizeof(ot_torrent) * ( end - match - 1 ) ); - if( ( --vector->size * OT_VECTOR_SHRINK_THRESH < vector->space ) && ( vector->space > OT_VECTOR_MIN_MEMBERS ) ) { - vector->space /= OT_VECTOR_SHRINK_RATIO; - vector->data = realloc( vector->data, vector->space * sizeof( ot_torrent ) ); - } -} - ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC_PARAM( int from_changeset ) ) { int exactmatch; ot_torrent *torrent; ot_peer *peer_dest; - ot_vector *torrents_list = lock_bucket_by_hash( hash ), *peer_pool; + ot_vector *torrents_list = mutex_bucket_lock_by_hash( hash ), *peer_pool; int base_pool = 0; #ifdef WANT_ACCESS_CONTROL @@ -184,13 +52,17 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC exactmatch = !exactmatch; #endif - if( exactmatch ) - return unlock_bucket_by_hash( hash ); + if( exactmatch ) { + mutex_bucket_unlock_by_hash( hash ); + return NULL; + } #endif torrent = vector_find_or_insert( torrents_list, (void*)hash, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); - if( !torrent ) - return unlock_bucket_by_hash( hash ); + if( !torrent ) { + mutex_bucket_unlock_by_hash( hash ); + return NULL; + } if( !exactmatch ) { /* Create a new torrent entry, then */ @@ -198,7 +70,8 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC if( !( torrent->peer_list = malloc( sizeof (ot_peerlist) ) ) ) { vector_remove_torrent( torrents_list, torrent ); - return unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); + return NULL; } byte_zero( torrent->peer_list, sizeof( ot_peerlist ) ); @@ -216,7 +89,7 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC peer_pool = &torrent->peer_list->peers[0]; binary_search( peer, peer_pool->data, peer_pool->size, sizeof(ot_peer), OT_PEER_COMPARE_SIZE, &exactmatch ); if( exactmatch ) { - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return torrent; } base_pool = 1; @@ -248,7 +121,7 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC torrent->peer_list->seed_count--; case 1: default: torrent->peer_list->peer_count--; - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return torrent; } } @@ -269,7 +142,7 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC memmove( peer_dest, peer, sizeof( ot_peer ) ); } - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return torrent; } @@ -282,13 +155,13 @@ ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC size_t return_peers_for_torrent( ot_hash *hash, size_t amount, char *reply, int is_tcp ) { char *r = reply; int exactmatch; - ot_vector *torrents_list = lock_bucket_by_hash( hash ); + ot_vector *torrents_list = mutex_bucket_lock_by_hash( hash ); ot_torrent *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); ot_peerlist *peer_list = torrent->peer_list; size_t index; if( !torrent ) { - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return 0; } @@ -338,12 +211,12 @@ size_t return_peers_for_torrent( ot_hash *hash, size_t amount, char *reply, int if( is_tcp ) *r++ = 'e'; - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return r - reply; } /* Release memory we allocated too much */ -static void fix_mmapallocation( void *buf, size_t old_alloc, size_t new_alloc ) { +void fix_mmapallocation( void *buf, size_t old_alloc, size_t new_alloc ) { int page_size = getpagesize(); size_t old_pages = 1 + old_alloc / page_size; size_t new_pages = 1 + new_alloc / page_size; @@ -356,19 +229,23 @@ static void fix_mmapallocation( void *buf, size_t old_alloc, size_t new_alloc ) size_t return_fullscrape_for_tracker( char **reply ) { size_t torrent_count = 0, j; size_t allocated, replysize; - int i; + ot_vector *torrents_list; + int bucket; char *r; - for( i=0; isize; + mutex_bucket_unlock( bucket ); + } /* one extra for pro- and epilogue */ allocated = 100*(1+torrent_count); if( !( r = *reply = mmap( NULL, allocated, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ) ) ) return 0; memmove( r, "d5:filesd", 9 ); r += 9; - for( i=0; isize; ++j ) { ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[j] ).hash; @@ -378,6 +255,7 @@ size_t return_fullscrape_for_tracker( char **reply ) { r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count ); } } + mutex_bucket_unlock( bucket ); } *r++='e'; *r++='e'; @@ -388,45 +266,10 @@ size_t return_fullscrape_for_tracker( char **reply ) { return replysize; } -size_t return_memstat_for_tracker( char **reply ) { - size_t torrent_count = 0, j; - size_t allocated, replysize; - int i, k; - char *r; - - for( i=0; isize; - } - - allocated = OT_BUCKET_COUNT*32 + (43+OT_POOLS_COUNT*32)*torrent_count; - if( !( r = *reply = mmap( NULL, allocated, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ) ) ) return 0; - - for( i=0; isize; ++j ) { - ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; - ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[j] ).hash; - r += sprintf( r, "\n%s:\n", to_hex( hex_out, (ot_byte*)hash) ); - for( k=0; kpeers[k].size), (unsigned int)peer_list->peers[k].space ); - } - } - - replysize = ( r - *reply ); - fix_mmapallocation( *reply, allocated, replysize ); - - return replysize; -} - /* Fetches scrape info for a specific torrent */ size_t return_udp_scrape_for_torrent( ot_hash *hash, char *reply ) { int exactmatch; - ot_vector *torrents_list = lock_bucket_by_hash( hash ); + ot_vector *torrents_list = mutex_bucket_lock_by_hash( hash ); ot_torrent *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); if( !exactmatch ) { @@ -443,7 +286,7 @@ size_t return_udp_scrape_for_torrent( ot_hash *hash, char *reply ) { r[2] = htonl( torrent->peer_list->peer_count-torrent->peer_list->seed_count ); } } - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return 12; } @@ -456,7 +299,7 @@ size_t return_tcp_scrape_for_torrent( ot_hash *hash_list, int amount, char *repl for( i=0; idata, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); if( exactmatch ) { @@ -468,347 +311,22 @@ size_t return_tcp_scrape_for_torrent( ot_hash *hash_list, int amount, char *repl torrent->peer_list->seed_count, torrent->peer_list->down_count, torrent->peer_list->peer_count-torrent->peer_list->seed_count ) + 23; } } - unlock_bucket_by_hash( hash ); - } - - *r++ = 'e'; *r++ = 'e'; - return r - reply; -} - -#ifdef WANT_TRACKER_SYNC -/* Import Changeset from an external authority - format: d4:syncd[..]ee - [..]: ( 20:01234567890abcdefghij16:XXXXYYYY )+ -*/ -int add_changeset_to_tracker( ot_byte *data, size_t len ) { - ot_hash *hash; - ot_byte *end = data + len; - unsigned long peer_count; - - /* We do know, that the string is \n terminated, so it cant - overflow */ - if( byte_diff( data, 8, "d4:syncd" ) ) return -1; - data += 8; - - while( 1 ) { - if( byte_diff( data, 3, "20:" ) ) { - if( byte_diff( data, 2, "ee" ) ) - return -1; - return 0; - } - data += 3; - hash = (ot_hash*)data; - data += sizeof( ot_hash ); - - /* Scan string length indicator */ - data += ( len = scan_ulong( (char*)data, &peer_count ) ); - - /* If no long was scanned, it is not divisible by 8, it is not - followed by a colon or claims to need to much memory, we fail */ - if( !len || !peer_count || ( peer_count & 7 ) || ( *data++ != ':' ) || ( data + peer_count > end ) ) - return -1; - - while( peer_count > 0 ) { - add_peer_to_torrent( hash, (ot_peer*)data, 1 ); - data += 8; peer_count -= 8; - } - } - return 0; -} - -/* Proposed output format - d4:syncd20:8*N:(xxxxyyyy)*Nee -*/ -size_t return_changeset_for_tracker( char **reply ) { - size_t allocated = 0, i, replysize; - int bucket; - char *r; - - /* Maybe there is time to clean_all_torrents(); */ - - /* Determine space needed for whole changeset */ - for( bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket ) { - ot_vector *torrents_list = all_torrents + bucket; - for( i=0; isize; ++i ) { - ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; - allocated += sizeof( ot_hash ) + sizeof(ot_peer) * torrent->peer_list->changeset.size + 13; - } + mutex_bucket_unlock_by_hash( hash ); } - /* add "d4:syncd" and "ee" */ - allocated += 8 + 2; - - if( !( r = *reply = mmap( NULL, allocated, PROT_READ | PROT_WRITE, MAP_ANON | MAP_PRIVATE, -1, 0 ) ) ) - return 0; - - memmove( r, "d4:syncd", 8 ); r += 8; - for( bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket ) { - ot_vector *torrents_list = all_torrents + bucket; - for( i=0; isize; ++i ) { - ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; - const size_t byte_count = sizeof(ot_peer) * torrent->peer_list->changeset.size; - *r++ = '2'; *r++ = '0'; *r++ = ':'; - memmove( r, torrent->hash, sizeof( ot_hash ) ); r += sizeof( ot_hash ); - r += sprintf( r, "%zd:", byte_count ); - memmove( r, torrent->peer_list->changeset.data, byte_count ); r += byte_count; - } - } *r++ = 'e'; *r++ = 'e'; - - replysize = ( r - *reply ); - fix_mmapallocation( *reply, allocated, replysize ); - - return replysize; -} -#endif - -/* Clean a single torrent - return 1 if torrent timed out -*/ -static int clean_single_torrent( ot_torrent *torrent ) { - ot_peerlist *peer_list = torrent->peer_list; - size_t peers_count = 0, seeds_count; - time_t timedout = (int)( NOW - peer_list->base ); - int i; -#ifdef WANT_TRACKER_SYNC - char *new_peers; -#endif - - /* Torrent has idled out */ - if( timedout > OT_TORRENT_TIMEOUT ) - return 1; - - /* Nothing to be cleaned here? Test if torrent is worth keeping */ - if( timedout > OT_POOLS_COUNT ) { - if( !peer_list->peer_count ) - return peer_list->down_count ? 0 : 1; - timedout = OT_POOLS_COUNT; - } - - /* Release vectors that have timed out */ - for( i = OT_POOLS_COUNT - timedout; i < OT_POOLS_COUNT; ++i ) - free( peer_list->peers[i].data); - - /* Shift vectors back by the amount of pools that were shifted out */ - memmove( peer_list->peers + timedout, peer_list->peers, sizeof( ot_vector ) * ( OT_POOLS_COUNT - timedout ) ); - byte_zero( peer_list->peers, sizeof( ot_vector ) * timedout ); - - /* Shift back seed counts as well */ - memmove( peer_list->seed_counts + timedout, peer_list->seed_counts, sizeof( size_t ) * ( OT_POOLS_COUNT - timedout ) ); - byte_zero( peer_list->seed_counts, sizeof( size_t ) * timedout ); - -#ifdef WANT_TRACKER_SYNC - /* Save the block modified within last OT_POOLS_TIMEOUT */ - if( peer_list->peers[1].size && - ( new_peers = realloc( peer_list->changeset.data, sizeof( ot_peer ) * peer_list->peers[1].size ) ) ) - { - memmove( new_peers, peer_list->peers[1].data, peer_list->peers[1].size ); - peer_list->changeset.data = new_peers; - peer_list->changeset.size = sizeof( ot_peer ) * peer_list->peers[1].size; - } else { - free( peer_list->changeset.data ); - - memset( &peer_list->changeset, 0, sizeof( ot_vector ) ); - } -#endif - - peers_count = seeds_count = 0; - for( i = 0; i < OT_POOLS_COUNT; ++i ) { - peers_count += peer_list->peers[i].size; - seeds_count += peer_list->seed_counts[i]; - } - peer_list->seed_count = seeds_count; - peer_list->peer_count = peers_count; - - if( peers_count ) - peer_list->base = NOW; - else { - /* When we got here, the last time that torrent - has been touched is OT_POOLS_COUNT units before */ - peer_list->base = NOW - OT_POOLS_COUNT; - } - return 0; -} - -/* Clean up all peers in current bucket, remove timedout pools and - torrents */ -void clean_all_torrents( void ) { - ot_vector *torrents_list; - size_t i; - static int bucket; - ot_time time_now = NOW; - - /* Search for an uncleaned bucked */ - while( ( all_torrents_clean[bucket] == time_now ) && ( ++bucket < OT_BUCKET_COUNT ) ); - if( bucket >= OT_BUCKET_COUNT ) { - bucket = 0; return; - } - - all_torrents_clean[bucket] = time_now; - - mutex_bucket_lock( bucket ); - torrents_list = all_torrents + bucket; - for( i=0; isize; ++i ) { - ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + i; - if( clean_single_torrent( torrent ) ) { - vector_remove_torrent( torrents_list, torrent ); - --i; continue; - } - } - mutex_bucket_unlock( bucket ); -} - -typedef struct { size_t val; ot_torrent * torrent; } ot_record; - -/* Fetches stats from tracker */ -size_t return_stats_for_tracker( char *reply, int mode ) { - size_t torrent_count = 0, peer_count = 0, seed_count = 0, j; - ot_record top5s[5], top5c[5]; - char *r = reply; - int bucket; - - byte_zero( top5s, sizeof( top5s ) ); - byte_zero( top5c, sizeof( top5c ) ); - - for( bucket=0; bucketsize; - for( j=0; jsize; ++j ) { - ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; - if( mode == STATS_TOP5 ) { - int idx = 4; while( (idx >= 0) && ( peer_list->peer_count > top5c[idx].val ) ) --idx; - if ( idx++ != 4 ) { - memmove( top5c + idx + 1, top5c + idx, ( 4 - idx ) * sizeof( ot_record ) ); - top5c[idx].val = peer_list->peer_count; - top5c[idx].torrent = (ot_torrent*)(torrents_list->data) + j; - } - idx = 4; while( (idx >= 0) && ( peer_list->seed_count > top5s[idx].val ) ) --idx; - if ( idx++ != 4 ) { - memmove( top5s + idx + 1, top5s + idx, ( 4 - idx ) * sizeof( ot_record ) ); - top5s[idx].val = peer_list->seed_count; - top5s[idx].torrent = (ot_torrent*)(torrents_list->data) + j; - } - } - peer_count += peer_list->peer_count; seed_count += peer_list->seed_count; - } - mutex_bucket_unlock( bucket ); - } - if( mode == STATS_TOP5 ) { - char hex_out[42]; - int idx; - r += sprintf( r, "Top5 torrents by peers:\n" ); - for( idx=0; idx<5; ++idx ) - if( top5c[idx].torrent ) - r += sprintf( r, "\t%zd\t%s\n", top5c[idx].val, to_hex( hex_out, top5c[idx].torrent->hash) ); - r += sprintf( r, "Top5 torrents by seeds:\n" ); - for( idx=0; idx<5; ++idx ) - if( top5s[idx].torrent ) - r += sprintf( r, "\t%zd\t%s\n", top5s[idx].val, to_hex( hex_out, top5s[idx].torrent->hash) ); - } else - r += sprintf( r, "%zd\n%zd\nopentracker serving %zd torrents\nopentracker", peer_count, seed_count, torrent_count ); - - return r - reply; -} - -/* This function collects 4096 /24s in 4096 possible - malloc blocks -*/ -size_t return_stats_for_slash24s( char *reply, size_t amount, ot_dword thresh ) { - -#define NUM_TOPBITS 12 -#define NUM_LOWBITS (24-NUM_TOPBITS) -#define NUM_BUFS (1<size; ++j ) { - ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[j] ).peer_list; - for( k=0; kpeers[k].data; - size_t numpeers = peer_list->peers[k].size; - for( l=0; l> 8; - ot_dword *count = counts[ s24 >> NUM_LOWBITS ]; - if( !count ) { - count = malloc( sizeof(ot_dword) * NUM_S24S ); - if( !count ) - goto bailout_cleanup; - byte_zero( count, sizeof( ot_dword ) * NUM_S24S ); - counts[ s24 >> NUM_LOWBITS ] = count; - } - count[ s24 & MSK_S24S ]++; - } - } - } - mutex_bucket_unlock( bucket ); - } - - k = l = 0; /* Debug: count allocated bufs */ - for( i=0; i < NUM_BUFS; ++i ) { - ot_dword *count = counts[i]; - if( !counts[i] ) - continue; - ++k; /* Debug: count allocated bufs */ - for( j=0; j < NUM_S24S; ++j ) { - if( count[j] > thresh ) { - /* This subnet seems to announce more torrents than the last in our list */ - int insert_pos = amount - 1; - while( ( insert_pos >= 0 ) && ( count[j] > slash24s[ 2 * insert_pos ] ) ) - --insert_pos; - ++insert_pos; - memmove( slash24s + 2 * ( insert_pos + 1 ), slash24s + 2 * ( insert_pos ), 2 * sizeof( ot_dword ) * ( amount - insert_pos - 1 ) ); - slash24s[ 2 * insert_pos ] = count[j]; - slash24s[ 2 * insert_pos + 1 ] = ( i << NUM_TOPBITS ) + j; - if( slash24s[ 2 * amount - 2 ] > thresh ) - thresh = slash24s[ 2 * amount - 2 ]; - } - if( count[j] ) ++l; - } - free( count ); - } - - r += sprintf( r, "Allocated bufs: %zd, used s24s: %zd\n", k, l ); - - for( i=0; i < amount; ++i ) - if( slash24s[ 2*i ] >= thresh ) { - ot_dword ip = slash24s[ 2*i +1 ]; - r += sprintf( r, "% 10ld %d.%d.%d.0/24\n", (long)slash24s[ 2*i ], (int)(ip >> 16), (int)(255 & ( ip >> 8 )), (int)(ip & 255) ); - } - return r - reply; - -bailout_cleanup: - - for( i=0; i < NUM_BUFS; ++i ) - free( counts[i] ); - - return 0; } size_t remove_peer_from_torrent( ot_hash *hash, ot_peer *peer, char *reply, int is_tcp ) { int exactmatch; size_t index; - ot_vector *torrents_list = lock_bucket_by_hash( hash ); + ot_vector *torrents_list = mutex_bucket_lock_by_hash( hash ); ot_torrent *torrent = binary_search( hash, torrents_list->data, torrents_list->size, sizeof( ot_torrent ), OT_HASH_COMPARE_SIZE, &exactmatch ); ot_peerlist *peer_list; if( !exactmatch ) { - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); if( is_tcp ) return sprintf( reply, "d8:completei0e10:incompletei0e8:intervali%ie5:peers0:e", OT_CLIENT_REQUEST_INTERVAL_RANDOM ); @@ -835,7 +353,7 @@ exit_loop: if( is_tcp ) { size_t reply_size = sprintf( reply, "d8:completei%zde10:incompletei%zde8:intervali%ie5:peers0:e", peer_list->seed_count, peer_list->peer_count - peer_list->seed_count, OT_CLIENT_REQUEST_INTERVAL_RANDOM ); - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return reply_size; } @@ -844,7 +362,7 @@ exit_loop: ((ot_dword*)reply)[3] = peer_list->peer_count - peer_list->seed_count; ((ot_dword*)reply)[4] = peer_list->seed_count; - unlock_bucket_by_hash( hash ); + mutex_bucket_unlock_by_hash( hash ); return (size_t)20; } @@ -874,30 +392,28 @@ int trackerlogic_init( const char * const serverdir ) { } srandom( time(NULL) ); - - /* Initialize control structures */ - byte_zero( all_torrents, sizeof( all_torrents ) ); - + + clean_init( ); mutex_init( ); return 0; } void trackerlogic_deinit( void ) { - int i; + int bucket; size_t j; /* Free all torrents... */ - for(i=0; isize ) { + for( j=0; jsize; ++j ) { + ot_torrent *torrent = ((ot_torrent*)(torrents_list->data)) + j; + free_peerlist( torrent->peer_list ); + } + free( torrents_list->data ); } } - byte_zero( all_torrents, sizeof (all_torrents)); - byte_zero( all_torrents_clean, sizeof (all_torrents_clean)); - mutex_deinit( ); -} + clean_deinit( ); +} \ No newline at end of file diff --git a/trackerlogic.h b/trackerlogic.h index 5b03012..fd80f92 100644 --- a/trackerlogic.h +++ b/trackerlogic.h @@ -1,8 +1,8 @@ /* This software was written by Dirk Engling It is considered beerware. Prost. Skol. Cheers or whatever. */ -#ifndef __TRACKERLOGIC_H__ -#define __TRACKERLOGIC_H__ +#ifndef __OT_TRACKERLOGIC_H__ +#define __OT_TRACKERLOGIC_H__ #include #include @@ -20,16 +20,6 @@ typedef ot_byte ot_hash[20]; typedef ot_dword ot_ip; typedef time_t ot_time; -#define OT_VECTOR_MIN_MEMBERS 4 -#define OT_VECTOR_GROW_RATIO 8 -#define OT_VECTOR_SHRINK_THRESH 6 -#define OT_VECTOR_SHRINK_RATIO 4 -typedef struct { - void *data; - size_t size; - size_t space; -} ot_vector; - /* Some tracker behaviour tunable */ #define OT_CLIENT_TIMEOUT 30 #define OT_CLIENT_TIMEOUT_CHECKINTERVAL 10 @@ -42,6 +32,10 @@ typedef struct { #define OT_CLIENT_REQUEST_INTERVAL_RANDOM ( OT_CLIENT_REQUEST_INTERVAL - OT_CLIENT_REQUEST_VARIATION/2 + (int)( random( ) % OT_CLIENT_REQUEST_VARIATION ) ) +/* We maintain a list of 1024 pointers to sorted list of ot_torrent structs + Sort key is, of course, its hash */ +#define OT_BUCKET_COUNT 1024 + /* Number of tracker admin ip addresses allowed */ #define OT_ADMINIP_MAX 64 #define OT_MAX_THREADS 16 @@ -70,7 +64,16 @@ static const ot_byte PEER_FLAG_STOPPED = 0x20; #define OT_PEER_COMPARE_SIZE ((size_t)6) #define OT_HASH_COMPARE_SIZE (sizeof(ot_hash)) +struct ot_peerlist; +typedef struct ot_peerlist ot_peerlist; typedef struct { + ot_hash hash; + ot_peerlist *peer_list; +} ot_torrent; + +#include "ot_vector.h" + +struct ot_peerlist { ot_time base; size_t seed_count; size_t peer_count; @@ -80,12 +83,7 @@ typedef struct { #ifdef WANT_TRACKER_SYNC ot_vector changeset; #endif -} ot_peerlist; - -typedef struct { - ot_hash hash; - ot_peerlist *peer_list; -} ot_torrent; +}; /* Exported functions @@ -100,27 +98,21 @@ typedef struct { int trackerlogic_init( const char * const serverdir ); void trackerlogic_deinit( void ); -enum { STATS_CONNS, STATS_PEERS, STATS_TOP5, STATS_DMEM, STATS_TCP, STATS_UDP, STATS_SLASH24S, SYNC_IN, SYNC_OUT, STATS_FULLSCRAPE }; - ot_torrent *add_peer_to_torrent( ot_hash *hash, ot_peer *peer WANT_TRACKER_SYNC_PARAM( int from_changeset ) ); size_t remove_peer_from_torrent( ot_hash *hash, ot_peer *peer, char *reply, int is_tcp ); size_t return_peers_for_torrent( ot_hash *hash, size_t amount, char *reply, int is_tcp ); size_t return_fullscrape_for_tracker( char **reply ); size_t return_tcp_scrape_for_torrent( ot_hash *hash, int amount, char *reply ); size_t return_udp_scrape_for_torrent( ot_hash *hash, char *reply ); -size_t return_stats_for_tracker( char *reply, int mode ); -size_t return_stats_for_slash24s( char *reply, size_t amount, ot_dword thresh ); -size_t return_memstat_for_tracker( char **reply ); void clean_all_torrents( void ); -#ifdef WANT_TRACKER_SYNC -size_t return_changeset_for_tracker( char **reply ); -int add_changeset_to_tracker( ot_byte *data, size_t len ); -#endif - #if defined ( WANT_BLACKLISTING ) || defined ( WANT_CLOSED_TRACKER ) int accesslist_addentry( ot_hash *hash ); void accesslist_reset( void ); #endif +/* Helper, before it moves to its own object */ +void fix_mmapallocation( void *buf, size_t old_alloc, size_t new_alloc ); +void free_peerlist( ot_peerlist *peer_list ); + #endif