mirror of git://erdgeist.org/opentracker
You can not select more than 25 topics
Topics must start with a letter or number, can include dashes ('-') and can be up to 35 characters long.
224 lines
7.3 KiB
224 lines
7.3 KiB
/* This software was written by Dirk Engling <erdgeist@erdgeist.org> |
|
It is considered beerware. Prost. Skol. Cheers or whatever. */ |
|
|
|
/* System */ |
|
#include <sys/uio.h> |
|
#include <stdio.h> |
|
#include <string.h> |
|
#include <pthread.h> |
|
#include <arpa/inet.h> |
|
#ifdef WANT_COMPRESSION_GZIP |
|
#include <zlib.h> |
|
#endif |
|
|
|
/* Libowfat */ |
|
#include "byte.h" |
|
#include "textcode.h" |
|
|
|
/* Opentracker */ |
|
#include "trackerlogic.h" |
|
#include "ot_mutex.h" |
|
#include "ot_iovec.h" |
|
#include "ot_fullscrape.h" |
|
|
|
/* Fetch full scrape info for all torrents |
|
Full scrapes usually are huge and one does not want to |
|
allocate more memory. So lets get them in 512k units |
|
*/ |
|
#define OT_SCRAPE_CHUNK_SIZE (512*1024) |
|
|
|
/* "d8:completei%zde10:downloadedi%zde10:incompletei%zdee" */ |
|
#define OT_SCRAPE_MAXENTRYLEN 256 |
|
|
|
#ifdef WANT_COMPRESSION_GZIP |
|
#define IF_COMPRESSION( TASK ) if( mode & TASK_FLAG_GZIP ) TASK |
|
#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) , param1, param2, param3 |
|
#else |
|
#define IF_COMPRESSION( TASK ) |
|
#define WANT_COMPRESSION_GZIP_PARAM( param1, param2, param3 ) |
|
#endif |
|
|
|
/* Forward declaration */ |
|
static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ); |
|
|
|
/* Converter function from memory to human readable hex strings |
|
XXX - Duplicated from ot_stats. Needs fix. */ |
|
static char*to_hex(char*d,ot_byte*s){char*m="0123456789ABCDEF";char *t=d;char*e=d+40;while(d<e){*d++=m[*s>>4];*d++=m[*s++&15];}*d=0;return t;} |
|
|
|
/* This is the entry point into this worker thread |
|
It grabs tasks from mutex_tasklist and delivers results back |
|
*/ |
|
static void * fullscrape_worker( void * args) { |
|
int iovec_entries; |
|
struct iovec *iovector; |
|
|
|
args = args; |
|
|
|
while( 1 ) { |
|
ot_tasktype tasktype = TASK_FULLSCRAPE; |
|
ot_taskid taskid = mutex_workqueue_poptask( &tasktype ); |
|
fullscrape_make( &iovec_entries, &iovector, tasktype ); |
|
if( mutex_workqueue_pushresult( taskid, iovec_entries, iovector ) ) |
|
iovec_free( &iovec_entries, &iovector ); |
|
} |
|
return NULL; |
|
} |
|
|
|
static pthread_t thread_id; |
|
void fullscrape_init( ) { |
|
pthread_create( &thread_id, NULL, fullscrape_worker, NULL ); |
|
} |
|
|
|
void fullscrape_deinit( ) { |
|
pthread_cancel( thread_id ); |
|
} |
|
|
|
void fullscrape_deliver( int64 socket, ot_tasktype tasktype ) { |
|
mutex_workqueue_pushtask( socket, tasktype ); |
|
} |
|
|
|
static int fullscrape_increase( int *iovec_entries, struct iovec **iovector, |
|
char **r, char **re WANT_COMPRESSION_GZIP_PARAM( z_stream *strm, ot_tasktype mode, int zaction ) ) { |
|
/* Allocate a fresh output buffer at the end of our buffers list */ |
|
if( !( *r = iovec_fix_increase_or_free( iovec_entries, iovector, *r, OT_SCRAPE_CHUNK_SIZE ) ) ) { |
|
|
|
/* Deallocate gzip buffers */ |
|
IF_COMPRESSION( deflateEnd(strm); ) |
|
|
|
/* Release lock on current bucket and return */ |
|
return -1; |
|
} |
|
|
|
/* Adjust new end of output buffer */ |
|
*re = *r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; |
|
|
|
/* When compressing, we have all the bytes in output buffer */ |
|
#ifdef WANT_COMPRESSION_GZIP |
|
if( mode & TASK_FLAG_GZIP ) { |
|
*re -= OT_SCRAPE_MAXENTRYLEN; |
|
strm->next_out = (ot_byte*)*r; |
|
strm->avail_out = OT_SCRAPE_CHUNK_SIZE; |
|
if( deflate( strm, zaction ) < Z_OK ) |
|
fprintf( stderr, "deflate() failed while in fullscrape_increase().\n" ); |
|
*r = (char*)strm->next_out; |
|
} |
|
#endif |
|
|
|
return 0; |
|
} |
|
|
|
static void fullscrape_make( int *iovec_entries, struct iovec **iovector, ot_tasktype mode ) { |
|
int bucket; |
|
char *r, *re; |
|
#ifdef WANT_COMPRESSION_GZIP |
|
char compress_buffer[OT_SCRAPE_MAXENTRYLEN]; |
|
z_stream strm; |
|
#endif |
|
|
|
/* Setup return vector... */ |
|
*iovec_entries = 0; |
|
*iovector = NULL; |
|
if( !( r = iovec_increase( iovec_entries, iovector, OT_SCRAPE_CHUNK_SIZE ) ) ) |
|
return; |
|
|
|
/* re points to low watermark */ |
|
re = r + OT_SCRAPE_CHUNK_SIZE - OT_SCRAPE_MAXENTRYLEN; |
|
|
|
#ifdef WANT_COMPRESSION_GZIP |
|
if( mode & TASK_FLAG_GZIP ) { |
|
re += OT_SCRAPE_MAXENTRYLEN; |
|
byte_zero( &strm, sizeof(strm) ); |
|
strm.next_in = (ot_byte*)compress_buffer; |
|
strm.next_out = (ot_byte*)r; |
|
strm.avail_out = OT_SCRAPE_CHUNK_SIZE; |
|
if( deflateInit2(&strm,9,Z_DEFLATED,31,8,Z_DEFAULT_STRATEGY) != Z_OK ) |
|
fprintf( stderr, "not ok.\n" ); |
|
r = compress_buffer; |
|
} |
|
#endif |
|
|
|
if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) |
|
r += sprintf( r, "d5:filesd" ); |
|
|
|
/* For each bucket... */ |
|
for( bucket=0; bucket<OT_BUCKET_COUNT; ++bucket ) { |
|
/* Get exclusive access to that bucket */ |
|
ot_vector *torrents_list = mutex_bucket_lock( bucket ); |
|
size_t tor_offset; |
|
|
|
/* For each torrent in this bucket.. */ |
|
for( tor_offset=0; tor_offset<torrents_list->size; ++tor_offset ) { |
|
/* Address torrents members */ |
|
ot_peerlist *peer_list = ( ((ot_torrent*)(torrents_list->data))[tor_offset] ).peer_list; |
|
ot_hash *hash =&( ((ot_torrent*)(torrents_list->data))[tor_offset] ).hash; |
|
|
|
switch( mode & TASK_TASK_MASK ) { |
|
case TASK_FULLSCRAPE: |
|
default: |
|
|
|
/* push hash as bencoded string */ |
|
*r++='2'; *r++='0'; *r++=':'; |
|
memmove( r, hash, 20 ); r+=20; |
|
|
|
/* push rest of the scrape string */ |
|
r += sprintf( r, "d8:completei%zde10:downloadedi%zde10:incompletei%zdee", peer_list->seed_count, peer_list->down_count, peer_list->peer_count-peer_list->seed_count ); |
|
|
|
break; |
|
case TASK_FULLSCRAPE_TPB_ASCII: |
|
to_hex( r, *hash ); r+=40; |
|
r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); |
|
break; |
|
case TASK_FULLSCRAPE_TPB_BINARY: |
|
memmove( r, hash, 20 ); r+=20; |
|
*(ot_dword*)r++ = htonl( (uint32_t)peer_list->seed_count ); |
|
*(ot_dword*)r++ = htonl( (uint32_t)( peer_list->peer_count-peer_list->seed_count) ); |
|
break; |
|
case TASK_FULLSCRAPE_TPB_URLENCODED: |
|
r += fmt_urlencoded( r, (char *)*hash, 20 ); |
|
r += sprintf( r, ":%zd:%zd\n", peer_list->seed_count, peer_list->peer_count-peer_list->seed_count ); |
|
break; |
|
} |
|
|
|
#ifdef WANT_COMPRESSION_GZIP |
|
if( mode & TASK_FLAG_GZIP ) { |
|
strm.next_in = (ot_byte*)compress_buffer; |
|
strm.avail_in = r - compress_buffer; |
|
if( deflate( &strm, Z_NO_FLUSH ) < Z_OK ) |
|
fprintf( stderr, "deflate() failed while in fullscrape_make().\n" ); |
|
r = (char*)strm.next_out; |
|
} |
|
#endif |
|
|
|
/* Check if there still is enough buffer left */ |
|
while( r >= re ) |
|
if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_NO_FLUSH ) ) ) |
|
return mutex_bucket_unlock( bucket ); |
|
|
|
IF_COMPRESSION( r = compress_buffer; ) |
|
} |
|
|
|
/* All torrents done: release lock on currenct bucket */ |
|
mutex_bucket_unlock( bucket ); |
|
} |
|
|
|
if( ( mode & TASK_TASK_MASK ) == TASK_FULLSCRAPE ) |
|
r += sprintf( r, "ee" ); |
|
|
|
#ifdef WANT_COMPRESSION_GZIP |
|
if( mode & TASK_FLAG_GZIP ) { |
|
strm.next_in = (ot_byte*)compress_buffer; |
|
strm.avail_in = r - compress_buffer; |
|
if( deflate( &strm, Z_FINISH ) < Z_OK ) |
|
fprintf( stderr, "deflate() failed while in fullscrape_make()'s endgame.\n" ); |
|
r = (char*)strm.next_out; |
|
|
|
while( r >= re ) |
|
if( fullscrape_increase( iovec_entries, iovector, &r, &re WANT_COMPRESSION_GZIP_PARAM( &strm, mode, Z_FINISH ) ) ) |
|
return mutex_bucket_unlock( bucket ); |
|
deflateEnd(&strm); |
|
} |
|
#endif |
|
|
|
/* Release unused memory in current output buffer */ |
|
iovec_fixlast( iovec_entries, iovector, r ); |
|
}
|
|
|