Makefile

@@ -27,11 +27,6 @@ STRIP?=strip
 #FEATURES+=-DWANT_IP_FROM_QUERY_STRING
 FEATURES+=-DWANT_COMPRESSION_GZIP
 FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
-
-#FEATURES+=-DWANT_COMPRESSION_ZSTD
-#FEATURES+=-DWANT_COMPRESSION_ZSTD_ALWAYS
-#LDFLAGS+=-lzstd
-
 #FEATURES+=-DWANT_LOG_NETWORKS
 #FEATURES+=-DWANT_RESTRICT_STATS
 #FEATURES+=-DWANT_IP_FROM_PROXY

man1/opentracker.1

@@ -1,142 +0,0 @@
-.Dd 15/4/2024
-.Dt opentracker 1
-.Os Unix
-.Sh opentracker
-.Nm opentracker
-.Nd a free and open bittorrent tracker
-.Sh SYNOPSIS
-.Nm
-.Op Fl f Ar config
-.Op Fl i Ar ip-select
-.Op Fl p Ar port-bind-tcp
-.Op Fl P Ar port-bind-udp
-.Op Fl A Ar blessed-ip
-.Op Fl r Ar redirect-url
-.Op Fl d Ar chdir
-.Op Fl u Ar user
-.Op Fl w| Fl b accesslist
-.Sh DESCRIPTION
-.Nm
-is a bittorrent tracker that implements announce and scrape actions over the
-UDP and the plain http protocol, aiming for minimal resource usage.
-.Pp
-
-When invoked with parameters, it binds to TCP and UDP port 6969 on all
-interfaces. The recommended way to configure opentracker is by providing a
-config file using the
-.Op Fl f Ar config
-option. See
-.Xr opentracker.conf 4
-for details.
-.Pp
-
-.Sh OPTIONS
-The following options are available:
-
-.Bl -tag -width -indent=8
-.It Fl f Ar config
-Parse a config file with a list of options. Consecutive command options
-will override options from the config file. See
-.Xr opentracker.conf 4
-for details.
-
-.It Fl i Ar ip-select
-Select an ip address that will be used with the next
-.Op Fl p
-or
-.Op Fl P
-command to actually bind to this address. Setting this option without any bind
-options in the config file or
-.Op Fl p
-or
-.Op Fl P
-commands will limit opentracker to only bind to this address.
-.It Fl p Ar port-bind-tcp
-Bind to the TCP port on the last preceding ip address set with the
-.Op Fl i ip-select
-option or to all available addresses if none has been set. Can be given multiple
-times.
-.It Fl P Ar port-bind-udp
-Bind to the UDP port on the last preceding ip address set with the
-.Op Fl i ip-select
-option or to all available addresses if none has been set. Can be given multiple
-times.
-.It Fl A Ar blessed-ip
-Set an ip address in IPv4 or IPv6 or a net in CIDR notation to bless the network
-for access to restricted resources.
-.It Fl r Ar redirect-url
-Set the URL that
-.Nm
-will redirect users to when the / address is requested via HTTP.
-.It Fl d Ar chdir
-Sets the directory
-.Nm
-will
-.Xr chroot 2
-to if ran as root or
-.Xr chdir 2
-to if ran as unprivileged user. Note that any accesslist files need to be
-relative to and within that directory.
-.It Fl u Ar user
-User to run
-.Nm
-under after all operations that need privileges have finished.
-.It Fl w Ar accesslist | Fl b Ar accesslist
-If
-.Nm
-has been compiled with the
-.B WANT_ACCESSLIST_BLACK
-or
-.Br WANT_ACCESSLIST_WHITE
-options, this option sets the location of the accesslist.
-.El
-
-.Sh EXAMPLES
-
-Start
-.Nm
-bound on UDP and TCP ports 6969 on IPv6 localhost.
-
-.Dl # ./opentracker -i ::1 -p 6969 -P 6969
-
-.Pp
-Start
-.Nm
-bound on UDP port 6868 and TCP port 6868 on IPv4 localhost and allow
-privileged access from the network 192.168/16 while redirecting
-HTTP clients accessing the root directory, which is not covered by the
-bittorrent tracker protocol, to https://my-trackersite.com/.
-
-.Dl # ./opentracker -i 192.168.0.4 -p 6868 -P 6969 -A 192.168/16 -r https://my-trackersite.com/
-
-The announce URLs are http://192.168.0.4:6868/announce and
-udp://192.168.0.4:6868/announce respectively.
-
-.Sh FILES
-.Bl -tag -width indent
-.It Pa opentracker.conf
-The
-.Nm
-config file.
-.El
-.Sh SEE ALSO
-.Xr opentracker.conf 4
-.Pp
-opentracker documentation
-.Lk https://erdgeist.org/arts/software/opentracker
-.Pp
-Bittorrent tracker protocol
-.Lk http://www.bittorrent.org/beps/bep_0015.html
-.Sh
-.Sh AUTHOR
-.An Dirk Engling
-.Aq Mt erdgeist@erdgeist.org .
-.Sh LICENSE
-This software is released under the Beerware License:
-
-Permission is hereby granted, free of charge, to any person obtaining a copy of this software
-and associated documentation files (the "Software"), to deal in the Software with the following
-terms and conditions:
-
-If you meet the author(s) someday, and you think this software is worth it, you can buy them
-a beer in return.

man4/opentracker.conf.4

@@ -1,86 +0,0 @@
-.Dd 2024-04-18
-.Dt opentracker.conf 5
-.Os Unix
-.Sh NAME
-.Nm opentracker.conf
-.Nd configuration file for opentracker
-.Sh SYNOPSIS
-.Nm
-.Sh DESCRIPTION
-The
-.Nm
-configuration file specifies various options for configuring the behavior of the opentracker program.
-.Pp
-Lines starting with '#' are comments and are ignored. Options are specified as 'keyword value' pairs.
-.Pp
-The following options are available:
-
-.Bl -tag -width ".It access.proxy" -compact
-.It listen.tcp_udp Ar address
-Specifies an address opentracker will listen on for both TCP and UDP connections. If none are specified, opentracker listens on 0.0.0.0:6969 by default. Can be added more than once.
-
-.It listen.tcp Ar address
-Specifies the address opentracker will listen on for TCP connections. Can be added more than once.
-
-.It listen.udp Ar address
-Specifies the address opentracker will listen on for UDP connections. Can be added more than once.
-
-.It listen.udp.workers Ar threads
-Specifies how many threads will be spawned to handle UDP connections. Defaults to 4.
-
-.It access.whitelist Ar path/to/whitelist
-Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode.
-
-.It access.blacklist Ar path/to/blacklist
-Specifies the path to the blacklist file containing all torrent hashes that opentracker will not serve. Use this option if opentracker was compiled to allow blacklisting.
-
-.It access.fifo_add Ar path/to/adder.fifo
-Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be added to the main accesslist file.
-
-.It access.fifo_delete Ar path/to/deleter.fifo
-Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be removed from the main accesslist file.
-
-.It access.stats Ar ip_address_or_network
-Specifies the IP address or network in CIDR notation allowed to fetch stats from opentracker.
-
-.It access.stats_path Ar path
-Specifies the path to the stats location. You can configure opentracker to appear anywhere on your tracker. Defaults to /stats.
-
-.It access.proxy Ar ip_address_or_network
-Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once.
-
-.It livesync.cluster.listen Ar ip_address:port
-Specifies the IP address and port opentracker will listen on for incoming live sync packets to keep a cluster of opentrackers synchronized.
-
-.It livesync.cluster.node_ip Ar ip_address
-Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once.
-
-.It batchsync.cluster.admin_ip Ar ip_address
-Specifies the admin IP address for old-style (HTTP-based) asynchronous tracker syncing.
-
-.It tracker.rootdir Ar path
-Specifies the directory opentracker will chroot/chdir to. All black/white list files must be located in this directory.
-
-.It tracker.user Ar username
-Specifies the user opentracker will setuid to after binding to potentially privileged ports.
-
-.It tracker.redirect_url Ar URL
-Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request.
-
-.Sh EXAMPLES
-To specify the address opentracker will listen on for both TCP and UDP connections:
-.Dl listen.tcp_udp 0.0.0.0:6969
-.Pp
-To specify the address opentracker will listen on for TCP connections:
-.Dl listen.tcp 0.0.0.0
-.Pp
-To specify the address opentracker will listen on for UDP connections:
-.Dl listen.udp 0.0.0.0:6969
-
-.Sh SEE ALSO
-.Xr opentracker 1
-
-.Sh AUTHOR
-.An Dirk Engling
-.Aq Mt erdgeist@erdgeist.org
-

opentracker.conf.sample

@@ -2,7 +2,7 @@
 #
 
 # I)   Address opentracker will listen on, using both, tcp AND udp family
-#      (note, that port 6969 is implicit if omitted).
+#      (note, that port 6969 is implicite if ommitted).
 #
 #      If no listen option is given (here or on the command line), opentracker
 #      listens on 0.0.0.0:6969 tcp and udp.
@@ -83,10 +83,9 @@
 # IIb)
 #      If you do not want to grant anyone access to your stats, enable the
 #      WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
-#      or network allowed to fetch stats here.
+#      allowed to fetch stats here.
 #
 # access.stats 192.168.0.23
-# access.stats 10.1.1.23
 #
 #      There is another way of hiding your stats. You can obfuscate the path
 #      to them. Normally it is located at /stats but you can configure it to

ot_fullscrape.c

@@ -14,10 +14,6 @@
 #ifdef WANT_COMPRESSION_GZIP
 #include <zlib.h>
 #endif
-#ifdef WANT_COMPRESSION_ZSTD
-#include <zstd.h>
-#endif
-
 
 /* Libowfat */
 #include "byte.h"
@@ -44,9 +40,6 @@ static void fullscrape_make(int taskid, ot_tasktype mode);
 #ifdef WANT_COMPRESSION_GZIP
 static void fullscrape_make_gzip(int taskid, ot_tasktype mode);
 #endif
-#ifdef WANT_COMPRESSION_ZSTD
-static void fullscrape_make_zstd(int taskid, ot_tasktype mode);
-#endif
 
 /* Converter function from memory to human readable hex strings
    XXX - Duplicated from ot_stats. Needs fix. */
@@ -71,11 +64,6 @@ static void *fullscrape_worker(void *args) {
   while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_FULLSCRAPE;
     ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
-#ifdef WANT_COMPRESSION_ZSTD
-    if (tasktype & TASK_FLAG_ZSTD)
-      fullscrape_make_zstd(taskid, tasktype);
-    else
-#endif
 #ifdef WANT_COMPRESSION_GZIP
     if (tasktype & TASK_FLAG_GZIP)
       fullscrape_make_gzip(taskid, tasktype);
@@ -217,6 +205,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   struct iovec iovector = {NULL, 0};
   int          zres;
   z_stream     strm;
+  fprintf(stderr, "GZIP path\n");
   /* Setup return vector... */
   iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
   if (!iovector.iov_base)
@@ -278,11 +267,9 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
     mutex_bucket_unlock(bucket, 0);
 
     /* Parent thread died? */
-    if (!g_opentracker_running) {
+    if (!g_opentracker_running)
-      deflateEnd(&strm);
       return;
   }
-  }
 
   if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
     strm.next_in  = (uint8_t *)"ee";
@@ -295,8 +282,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
   if (mutex_workqueue_pushchunked(taskid, &iovector)) {
     free(iovector.iov_base);
-    deflateEnd(&strm);
+    return mutex_bucket_unlock(bucket, 0);
-    return;
   }
 
   /* Check if there's a last batch of data in the zlib buffer */
@@ -307,7 +293,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
       if (!iovector.iov_base) {
         fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
         deflateEnd(&strm);
-        return;
+        return mutex_bucket_unlock(bucket, 0);
       }
       strm.next_out  = iovector.iov_base;
       strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
@@ -325,133 +311,5 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
 /* WANT_COMPRESSION_GZIP */
 #endif
 
-#ifdef WANT_COMPRESSION_ZSTD
-
-static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
-  int            bucket;
-  char          *r;
-  struct iovec   iovector = {NULL, 0};
-  ZSTD_CCtx     *zstream = ZSTD_createCCtx();
-  ZSTD_inBuffer  inbuf;
-  ZSTD_outBuffer outbuf;
-  size_t         more_bytes;
-
-  if (!zstream)
-    return;
-
-  /* Setup return vector... */
-  iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
-  if (!iovector.iov_base) {
-    ZSTD_freeCCtx(zstream);
-    return;
-  }
-
-  /* Working with a compression level 6 is half as fast as level 3, but
-     seems to be the last reasonable bump that's worth extra cpu */
-  ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6);
-
-  outbuf.dst  = iovector.iov_base;
-  outbuf.size = OT_SCRAPE_CHUNK_SIZE;
-  outbuf.pos  = 0;
-
-  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
-    inbuf.src  = (const void *)"d5:filesd";
-    inbuf.size = strlen("d5:filesd");
-    inbuf.pos  = 0;
-    ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
-  }
-
-  /* For each bucket... */
-  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
-    /* Get exclusive access to that bucket */
-    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
-    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
-    size_t      i;
-
-    /* For each torrent in this bucket.. */
-    for (i = 0; i < torrents_list->size; ++i) {
-      char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
-      r             = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash);
-      inbuf.src     = compress_buffer;
-      inbuf.size    = r - compress_buffer;
-      inbuf.pos     = 0;
-      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
-
-      /* Check if there still is enough buffer left */
-      while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) {
-        iovector.iov_len = outbuf.size;
-
-        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
-          free(iovector.iov_base);
-          ZSTD_freeCCtx(zstream);
-          return mutex_bucket_unlock(bucket, 0);
-        }
-        /* Allocate a fresh output buffer */
-        iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
-        if (!iovector.iov_base) {
-          fprintf(stderr, "Out of memory trying to claim ouput buffer\n");
-          ZSTD_freeCCtx(zstream);
-          return mutex_bucket_unlock(bucket, 0);
-        }
-
-        outbuf.dst  = iovector.iov_base;
-        outbuf.size = OT_SCRAPE_CHUNK_SIZE;
-        outbuf.pos  = 0;
-
-        ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
-      }
-    }
-
-    /* All torrents done: release lock on current bucket */
-    mutex_bucket_unlock(bucket, 0);
-
-    /* Parent thread died? */
-    if (!g_opentracker_running)
-      return;
-  }
-
-  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
-    inbuf.src  = (const void *)"ee";
-    inbuf.size = strlen("ee");
-    inbuf.pos  = 0;
-  }
-
-  more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
-
-  iovector.iov_len = outbuf.pos;
-  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
-    free(iovector.iov_base);
-    ZSTD_freeCCtx(zstream);
-    return;
-  }
-
-  /* Check if there's a last batch of data in the zlib buffer */
-  if (more_bytes) {
-      /* Allocate a fresh output buffer */
-      iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
-
-      if (!iovector.iov_base) {
-        fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
-        ZSTD_freeCCtx(zstream);
-        return;
-      }
-
-      outbuf.dst  = iovector.iov_base;
-      outbuf.size = OT_SCRAPE_CHUNK_SIZE;
-      outbuf.pos  = 0;
-
-      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
-
-      /* Only pass the new buffer if there actually was some data left in the buffer */
-      iovector.iov_len = outbuf.pos;
-      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
-        free(iovector.iov_base);
-  }
-
-  ZSTD_freeCCtx(zstream);
-}
-/* WANT_COMPRESSION_ZSTD */
-#endif
-
 /* WANT_FULLSCRAPE */
 #endif

ot_http.c

@@ -159,15 +159,13 @@ ssize_t http_sendiovecdata(const int64 sock, struct ot_workstruct *ws, int iovec
 
   if (iovec_entries) {
 
-    if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD)
+    if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
-      encoding = "Content-Encoding: zstd\r\n";
-    else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
       encoding = "Content-Encoding: gzip\r\n";
     else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2)
       encoding = "Content-Encoding: bzip2\r\n";
 
     if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED))
-      header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size);
+      header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n%sContent-Length: %zd\r\n\r\n", encoding, size);
     else {
       if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) {
         header_size =
@@ -371,33 +369,18 @@ static ssize_t http_handle_fullscrape(const int64 sock, struct ot_workstruct *ws
   }
 #endif
 
-
-#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD)
-  ws->request[ws->request_size - 1] = 0;
 #ifdef WANT_COMPRESSION_GZIP
+  ws->request[ws->request_size - 1] = 0;
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
   if (strstr(ws->request, "gzip")) {
-    cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
-    format       |= TASK_FLAG_GZIP;
-  }
-#endif
-#ifdef WANT_COMPRESSION_ZSTD
-  if (strstr(ws->request, "zstd")) {
-    cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
-    format       |= TASK_FLAG_ZSTD;
-  }
-#endif
-
-#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS)
-  cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
-  format       |= TASK_FLAG_ZSTD;
 #endif
-
-#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS)
     cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
-  format       |= TASK_FLAG_GZIP;
+    format        = TASK_FLAG_GZIP;
+    stats_issue_event(EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip);
+#ifndef WANT_COMPRESSION_GZIP_ALWAYS
+  } else
 #endif
 #endif
-
     stats_issue_event(EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip);
 
 #ifdef _DEBUG_HTTPERROR

ot_http.h

@@ -10,9 +10,8 @@ typedef enum {
   STRUCT_HTTP_FLAG_WAITINGFORTASK      = 1,
   STRUCT_HTTP_FLAG_GZIP                = 2,
   STRUCT_HTTP_FLAG_BZIP2               = 4,
-  STRUCT_HTTP_FLAG_ZSTD                = 8,
+  STRUCT_HTTP_FLAG_CHUNKED             = 8,
-  STRUCT_HTTP_FLAG_CHUNKED             = 16,
+  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 16
-  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32
 } STRUCT_HTTP_FLAG;
 
 struct http_data {

ot_mutex.h

@@ -59,8 +59,7 @@ typedef enum {
 
   TASK_FLAG_GZIP                 = 0x1000,
   TASK_FLAG_BZIP2                = 0x2000,
-  TASK_FLAG_ZSTD                 = 0x4000,
+  TASK_FLAG_CHUNKED              = 0x4000,
-  TASK_FLAG_CHUNKED              = 0x8000,
 
   TASK_TASK_MASK                 = 0x0fff,
   TASK_CLASS_MASK                = 0x0f00,

ot_stats.h

@@ -19,7 +19,6 @@ typedef enum {
   EVENT_SCRAPE,
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
-  EVENT_FULLSCRAPE_REQUEST_ZSTD,
   EVENT_FULLSCRAPE, /* TCP only */
   EVENT_FAILED,
   EVENT_BUCKET_LOCKED,