Better point out what options can be given multiple times

Makefile

@@ -27,6 +27,11 @@ STRIP?=strip
 #FEATURES+=-DWANT_IP_FROM_QUERY_STRING
 FEATURES+=-DWANT_COMPRESSION_GZIP
 FEATURES+=-DWANT_COMPRESSION_GZIP_ALWAYS
+
+#FEATURES+=-DWANT_COMPRESSION_ZSTD
+#FEATURES+=-DWANT_COMPRESSION_ZSTD_ALWAYS
+#LDFLAGS+=-lzstd
+
 #FEATURES+=-DWANT_LOG_NETWORKS
 #FEATURES+=-DWANT_RESTRICT_STATS
 #FEATURES+=-DWANT_IP_FROM_PROXY

man1/opentracker.1

@@ -0,0 +1,142 @@
+.Dd 15/4/2024
+.Dt opentracker 1
+.Os Unix
+.Sh opentracker
+.Nm opentracker
+.Nd a free and open bittorrent tracker
+.Sh SYNOPSIS
+.Nm
+.Op Fl f Ar config
+.Op Fl i Ar ip-select
+.Op Fl p Ar port-bind-tcp
+.Op Fl P Ar port-bind-udp
+.Op Fl A Ar blessed-ip
+.Op Fl r Ar redirect-url
+.Op Fl d Ar chdir
+.Op Fl u Ar user
+.Op Fl w| Fl b accesslist
+.Sh DESCRIPTION
+.Nm
+is a bittorrent tracker that implements announce and scrape actions over the
+UDP and the plain http protocol, aiming for minimal resource usage.
+.Pp
+
+When invoked with parameters, it binds to TCP and UDP port 6969 on all
+interfaces. The recommended way to configure opentracker is by providing a
+config file using the
+.Op Fl f Ar config
+option. See
+.Xr opentracker.conf 4
+for details.
+.Pp
+
+.Sh OPTIONS
+The following options are available:
+
+.Bl -tag -width -indent=8
+.It Fl f Ar config
+Parse a config file with a list of options. Consecutive command options
+will override options from the config file. See
+.Xr opentracker.conf 4
+for details.
+
+.It Fl i Ar ip-select
+Select an ip address that will be used with the next
+.Op Fl p
+or
+.Op Fl P
+command to actually bind to this address. Setting this option without any bind
+options in the config file or
+.Op Fl p
+or
+.Op Fl P
+commands will limit opentracker to only bind to this address.
+.It Fl p Ar port-bind-tcp
+Bind to the TCP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl P Ar port-bind-udp
+Bind to the UDP port on the last preceding ip address set with the
+.Op Fl i ip-select
+option or to all available addresses if none has been set. Can be given multiple
+times.
+.It Fl A Ar blessed-ip
+Set an ip address in IPv4 or IPv6 or a net in CIDR notation to bless the network
+for access to restricted resources.
+.It Fl r Ar redirect-url
+Set the URL that
+.Nm
+will redirect users to when the / address is requested via HTTP.
+.It Fl d Ar chdir
+Sets the directory
+.Nm
+will
+.Xr chroot 2
+to if ran as root or
+.Xr chdir 2
+to if ran as unprivileged user. Note that any accesslist files need to be
+relative to and within that directory.
+.It Fl u Ar user
+User to run
+.Nm
+under after all operations that need privileges have finished.
+.It Fl w Ar accesslist | Fl b Ar accesslist
+If
+.Nm
+has been compiled with the
+.B WANT_ACCESSLIST_BLACK
+or
+.Br WANT_ACCESSLIST_WHITE
+options, this option sets the location of the accesslist.
+.El
+
+.Sh EXAMPLES
+
+Start
+.Nm
+bound on UDP and TCP ports 6969 on IPv6 localhost.
+
+.Dl # ./opentracker -i ::1 -p 6969 -P 6969
+
+.Pp
+Start
+.Nm
+bound on UDP port 6868 and TCP port 6868 on IPv4 localhost and allow
+privileged access from the network 192.168/16 while redirecting
+HTTP clients accessing the root directory, which is not covered by the
+bittorrent tracker protocol, to https://my-trackersite.com/.
+
+.Dl # ./opentracker -i 192.168.0.4 -p 6868 -P 6969 -A 192.168/16 -r https://my-trackersite.com/
+
+The announce URLs are http://192.168.0.4:6868/announce and
+udp://192.168.0.4:6868/announce respectively.
+
+.Sh FILES
+.Bl -tag -width indent
+.It Pa opentracker.conf
+The
+.Nm
+config file.
+.El
+.Sh SEE ALSO
+.Xr opentracker.conf 4
+.Pp
+opentracker documentation
+.Lk https://erdgeist.org/arts/software/opentracker
+.Pp
+Bittorrent tracker protocol
+.Lk http://www.bittorrent.org/beps/bep_0015.html
+.Sh
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org .
+.Sh LICENSE
+This software is released under the Beerware License:
+
+Permission is hereby granted, free of charge, to any person obtaining a copy of this software
+and associated documentation files (the "Software"), to deal in the Software with the following
+terms and conditions:
+
+If you meet the author(s) someday, and you think this software is worth it, you can buy them
+a beer in return.

man4/opentracker.conf.4

@@ -0,0 +1,86 @@
+.Dd 2024-04-18
+.Dt opentracker.conf 5
+.Os Unix
+.Sh NAME
+.Nm opentracker.conf
+.Nd configuration file for opentracker
+.Sh SYNOPSIS
+.Nm
+.Sh DESCRIPTION
+The
+.Nm
+configuration file specifies various options for configuring the behavior of the opentracker program.
+.Pp
+Lines starting with '#' are comments and are ignored. Options are specified as 'keyword value' pairs.
+.Pp
+The following options are available:
+
+.Bl -tag -width ".It access.proxy" -compact
+.It listen.tcp_udp Ar address
+Specifies an address opentracker will listen on for both TCP and UDP connections. If none are specified, opentracker listens on 0.0.0.0:6969 by default. Can be added more than once.
+
+.It listen.tcp Ar address
+Specifies the address opentracker will listen on for TCP connections. Can be added more than once.
+
+.It listen.udp Ar address
+Specifies the address opentracker will listen on for UDP connections. Can be added more than once.
+
+.It listen.udp.workers Ar threads
+Specifies how many threads will be spawned to handle UDP connections. Defaults to 4.
+
+.It access.whitelist Ar path/to/whitelist
+Specifies the path to the whitelist file containing all torrent hashes that opentracker will serve. Use this option if opentracker runs in a non-open mode.
+
+.It access.blacklist Ar path/to/blacklist
+Specifies the path to the blacklist file containing all torrent hashes that opentracker will not serve. Use this option if opentracker was compiled to allow blacklisting.
+
+.It access.fifo_add Ar path/to/adder.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be added to the main accesslist file.
+
+.It access.fifo_delete Ar path/to/deleter.fifo
+Specifies the path to the FIFO (named pipe) used for dynamic changesets to accesslists. Info hashes written to this FIFO will be removed from the main accesslist file.
+
+.It access.stats Ar ip_address_or_network
+Specifies the IP address or network in CIDR notation allowed to fetch stats from opentracker.
+
+.It access.stats_path Ar path
+Specifies the path to the stats location. You can configure opentracker to appear anywhere on your tracker. Defaults to /stats.
+
+.It access.proxy Ar ip_address_or_network
+Specifies the IP address or network of the reverse proxies. Opentracker will take the X-Forwarded-For address instead of the source IP address. Can be added more than once.
+
+.It livesync.cluster.listen Ar ip_address:port
+Specifies the IP address and port opentracker will listen on for incoming live sync packets to keep a cluster of opentrackers synchronized.
+
+.It livesync.cluster.node_ip Ar ip_address
+Specifies one trusted IP address for sync between trackers running in a cluster. Can be added more than once.
+
+.It batchsync.cluster.admin_ip Ar ip_address
+Specifies the admin IP address for old-style (HTTP-based) asynchronous tracker syncing.
+
+.It tracker.rootdir Ar path
+Specifies the directory opentracker will chroot/chdir to. All black/white list files must be located in this directory.
+
+.It tracker.user Ar username
+Specifies the user opentracker will setuid to after binding to potentially privileged ports.
+
+.It tracker.redirect_url Ar URL
+Specifies the URL opentracker will redirect to in response to a "GET / HTTP" request.
+
+.Sh EXAMPLES
+To specify the address opentracker will listen on for both TCP and UDP connections:
+.Dl listen.tcp_udp 0.0.0.0:6969
+.Pp
+To specify the address opentracker will listen on for TCP connections:
+.Dl listen.tcp 0.0.0.0
+.Pp
+To specify the address opentracker will listen on for UDP connections:
+.Dl listen.udp 0.0.0.0:6969
+
+.Sh SEE ALSO
+.Xr opentracker 1
+
+.Sh AUTHOR
+.An Dirk Engling
+.Aq Mt erdgeist@erdgeist.org
+

opentracker.conf.sample

@@ -2,7 +2,7 @@
 #
 
 # I)   Address opentracker will listen on, using both, tcp AND udp family
-#      (note, that port 6969 is implicite if ommitted).
+#      (note, that port 6969 is implicit if omitted).
 #
 #      If no listen option is given (here or on the command line), opentracker
 #      listens on 0.0.0.0:6969 tcp and udp.
@@ -83,9 +83,10 @@
 # IIb)
 #      If you do not want to grant anyone access to your stats, enable the
 #      WANT_RESTRICT_STATS option in Makefile and bless the ip addresses
-#      allowed to fetch stats here.
+#      or network allowed to fetch stats here.
 #
 # access.stats 192.168.0.23
+# access.stats 10.1.1.23
 #
 #      There is another way of hiding your stats. You can obfuscate the path
 #      to them. Normally it is located at /stats but you can configure it to

ot_fullscrape.c

@@ -14,6 +14,10 @@
 #ifdef WANT_COMPRESSION_GZIP
 #include <zlib.h>
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+#include <zstd.h>
+#endif
+
 
 /* Libowfat */
 #include "byte.h"
@@ -40,6 +44,9 @@ static void fullscrape_make(int taskid, ot_tasktype mode);
 #ifdef WANT_COMPRESSION_GZIP
 static void fullscrape_make_gzip(int taskid, ot_tasktype mode);
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode);
+#endif
 
 /* Converter function from memory to human readable hex strings
    XXX - Duplicated from ot_stats. Needs fix. */
@@ -64,6 +71,11 @@ static void *fullscrape_worker(void *args) {
   while (g_opentracker_running) {
     ot_tasktype tasktype = TASK_FULLSCRAPE;
     ot_taskid   taskid   = mutex_workqueue_poptask(&tasktype);
+#ifdef WANT_COMPRESSION_ZSTD
+    if (tasktype & TASK_FLAG_ZSTD)
+      fullscrape_make_zstd(taskid, tasktype);
+    else
+#endif
 #ifdef WANT_COMPRESSION_GZIP
     if (tasktype & TASK_FLAG_GZIP)
       fullscrape_make_gzip(taskid, tasktype);
@@ -205,7 +217,6 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   struct iovec iovector = {NULL, 0};
   int          zres;
   z_stream     strm;
-  fprintf(stderr, "GZIP path\n");
   /* Setup return vector... */
   iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
   if (!iovector.iov_base)
@@ -267,8 +278,10 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
     mutex_bucket_unlock(bucket, 0);
 
     /* Parent thread died? */
-    if (!g_opentracker_running)
+    if (!g_opentracker_running) {
+      deflateEnd(&strm);
       return;
+    }
   }
 
   if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
@@ -282,7 +295,8 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
   iovector.iov_len = (char *)strm.next_out - (char *)iovector.iov_base;
   if (mutex_workqueue_pushchunked(taskid, &iovector)) {
     free(iovector.iov_base);
-    return mutex_bucket_unlock(bucket, 0);
+    deflateEnd(&strm);
+    return;
   }
 
   /* Check if there's a last batch of data in the zlib buffer */
@@ -293,7 +307,7 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
       if (!iovector.iov_base) {
         fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
         deflateEnd(&strm);
-        return mutex_bucket_unlock(bucket, 0);
+        return;
       }
       strm.next_out  = iovector.iov_base;
       strm.avail_out = OT_SCRAPE_CHUNK_SIZE;
@@ -311,5 +325,133 @@ static void fullscrape_make_gzip(int taskid, ot_tasktype mode) {
 /* WANT_COMPRESSION_GZIP */
 #endif
 
+#ifdef WANT_COMPRESSION_ZSTD
+
+static void fullscrape_make_zstd(int taskid, ot_tasktype mode) {
+  int            bucket;
+  char          *r;
+  struct iovec   iovector = {NULL, 0};
+  ZSTD_CCtx     *zstream = ZSTD_createCCtx();
+  ZSTD_inBuffer  inbuf;
+  ZSTD_outBuffer outbuf;
+  size_t         more_bytes;
+
+  if (!zstream)
+    return;
+
+  /* Setup return vector... */
+  iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+  if (!iovector.iov_base) {
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Working with a compression level 6 is half as fast as level 3, but
+     seems to be the last reasonable bump that's worth extra cpu */
+  ZSTD_CCtx_setParameter(zstream, ZSTD_c_compressionLevel, 6);
+
+  outbuf.dst  = iovector.iov_base;
+  outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+  outbuf.pos  = 0;
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"d5:filesd";
+    inbuf.size = strlen("d5:filesd");
+    inbuf.pos  = 0;
+    ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+  }
+
+  /* For each bucket... */
+  for (bucket = 0; bucket < OT_BUCKET_COUNT; ++bucket) {
+    /* Get exclusive access to that bucket */
+    ot_vector  *torrents_list = mutex_bucket_lock(bucket);
+    ot_torrent *torrents      = (ot_torrent *)(torrents_list->data);
+    size_t      i;
+
+    /* For each torrent in this bucket.. */
+    for (i = 0; i < torrents_list->size; ++i) {
+      char compress_buffer[OT_SCRAPE_MAXENTRYLEN];
+      r             = fullscrape_write_one(mode, compress_buffer, torrents + i, &torrents[i].hash);
+      inbuf.src     = compress_buffer;
+      inbuf.size    = r - compress_buffer;
+      inbuf.pos     = 0;
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+
+      /* Check if there still is enough buffer left */
+      while (outbuf.pos + OT_SCRAPE_MAXENTRYLEN > outbuf.size) {
+        iovector.iov_len = outbuf.size;
+
+        if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+          free(iovector.iov_base);
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+        /* Allocate a fresh output buffer */
+        iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+        if (!iovector.iov_base) {
+          fprintf(stderr, "Out of memory trying to claim ouput buffer\n");
+          ZSTD_freeCCtx(zstream);
+          return mutex_bucket_unlock(bucket, 0);
+        }
+
+        outbuf.dst  = iovector.iov_base;
+        outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+        outbuf.pos  = 0;
+
+        ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_continue);
+      }
+    }
+
+    /* All torrents done: release lock on current bucket */
+    mutex_bucket_unlock(bucket, 0);
+
+    /* Parent thread died? */
+    if (!g_opentracker_running)
+      return;
+  }
+
+  if ((mode & TASK_TASK_MASK) == TASK_FULLSCRAPE) {
+    inbuf.src  = (const void *)"ee";
+    inbuf.size = strlen("ee");
+    inbuf.pos  = 0;
+  }
+
+  more_bytes = ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+  iovector.iov_len = outbuf.pos;
+  if (mutex_workqueue_pushchunked(taskid, &iovector)) {
+    free(iovector.iov_base);
+    ZSTD_freeCCtx(zstream);
+    return;
+  }
+
+  /* Check if there's a last batch of data in the zlib buffer */
+  if (more_bytes) {
+      /* Allocate a fresh output buffer */
+      iovector.iov_base = malloc(OT_SCRAPE_CHUNK_SIZE);
+
+      if (!iovector.iov_base) {
+        fprintf(stderr, "Problem with iovec_fix_increase_or_free\n");
+        ZSTD_freeCCtx(zstream);
+        return;
+      }
+
+      outbuf.dst  = iovector.iov_base;
+      outbuf.size = OT_SCRAPE_CHUNK_SIZE;
+      outbuf.pos  = 0;
+
+      ZSTD_compressStream2(zstream, &outbuf, &inbuf, ZSTD_e_end);
+
+      /* Only pass the new buffer if there actually was some data left in the buffer */
+      iovector.iov_len = outbuf.pos;
+      if (!iovector.iov_len || mutex_workqueue_pushchunked(taskid, &iovector))
+        free(iovector.iov_base);
+  }
+
+  ZSTD_freeCCtx(zstream);
+}
+/* WANT_COMPRESSION_ZSTD */
+#endif
+
 /* WANT_FULLSCRAPE */
 #endif

ot_http.c

@@ -159,13 +159,15 @@ ssize_t http_sendiovecdata(const int64 sock, struct ot_workstruct *ws, int iovec
 
   if (iovec_entries) {
 
-    if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
+    if (cookie->flag & STRUCT_HTTP_FLAG_ZSTD)
+      encoding = "Content-Encoding: zstd\r\n";
+    else if (cookie->flag & STRUCT_HTTP_FLAG_GZIP)
       encoding = "Content-Encoding: gzip\r\n";
     else if (cookie->flag & STRUCT_HTTP_FLAG_BZIP2)
       encoding = "Content-Encoding: bzip2\r\n";
 
     if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED))
-      header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: application/octet-stream\r\n%sContent-Length: %zd\r\n\r\n", encoding, size);
+      header_size = asprintf(&header, "HTTP/1.1 200 OK\r\nContent-Type: text/plain\r\n%sContent-Length: %zd\r\n\r\n", encoding, size);
     else {
       if (!(cookie->flag & STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER)) {
         header_size =
@@ -369,19 +371,34 @@ static ssize_t http_handle_fullscrape(const int64 sock, struct ot_workstruct *ws
   }
 #endif
 
-#ifdef WANT_COMPRESSION_GZIP
+
+#if defined(WANT_COMPRESSION_GZIP) || defined(WANT_COMPRESSION_ZSTD)
   ws->request[ws->request_size - 1] = 0;
-#ifndef WANT_COMPRESSION_GZIP_ALWAYS
+#ifdef WANT_COMPRESSION_GZIP
   if (strstr(ws->request, "gzip")) {
-#endif
     cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
-    format        = TASK_FLAG_GZIP;
-    stats_issue_event(EVENT_FULLSCRAPE_REQUEST_GZIP, 0, (uintptr_t)cookie->ip);
-#ifndef WANT_COMPRESSION_GZIP_ALWAYS
-  } else
+    format       |= TASK_FLAG_GZIP;
+  }
 #endif
+#ifdef WANT_COMPRESSION_ZSTD
+  if (strstr(ws->request, "zstd")) {
+    cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
+    format       |= TASK_FLAG_ZSTD;
+  }
+#endif
+
+#if defined(WANT_COMPRESSION_ZSTD) && defined(WANT_COMPRESSION_ZSTD_ALWAYS)
+  cookie->flag |= STRUCT_HTTP_FLAG_ZSTD;
+  format       |= TASK_FLAG_ZSTD;
 #endif
-    stats_issue_event(EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip);
+
+#if defined(WANT_COMPRESSION_GZIP) && defined(WANT_COMPRESSION_GZIP_ALWAYS)
+  cookie->flag |= STRUCT_HTTP_FLAG_GZIP;
+  format       |= TASK_FLAG_GZIP;
+#endif
+#endif
+
+  stats_issue_event(EVENT_FULLSCRAPE_REQUEST, 0, (uintptr_t)cookie->ip);
 
 #ifdef _DEBUG_HTTPERROR
   fprintf(stderr, "%s", ws->debugbuf);

ot_http.h

@@ -10,8 +10,9 @@ typedef enum {
   STRUCT_HTTP_FLAG_WAITINGFORTASK      = 1,
   STRUCT_HTTP_FLAG_GZIP                = 2,
   STRUCT_HTTP_FLAG_BZIP2               = 4,
-  STRUCT_HTTP_FLAG_CHUNKED             = 8,
-  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 16
+  STRUCT_HTTP_FLAG_ZSTD                = 8,
+  STRUCT_HTTP_FLAG_CHUNKED             = 16,
+  STRUCT_HTTP_FLAG_CHUNKED_IN_TRANSFER = 32
 } STRUCT_HTTP_FLAG;
 
 struct http_data {

ot_mutex.h

@@ -59,7 +59,8 @@ typedef enum {
 
   TASK_FLAG_GZIP                 = 0x1000,
   TASK_FLAG_BZIP2                = 0x2000,
-  TASK_FLAG_CHUNKED              = 0x4000,
+  TASK_FLAG_ZSTD                 = 0x4000,
+  TASK_FLAG_CHUNKED              = 0x8000,
 
   TASK_TASK_MASK                 = 0x0fff,
   TASK_CLASS_MASK                = 0x0f00,

ot_stats.h

@@ -19,6 +19,7 @@ typedef enum {
   EVENT_SCRAPE,
   EVENT_FULLSCRAPE_REQUEST,
   EVENT_FULLSCRAPE_REQUEST_GZIP,
+  EVENT_FULLSCRAPE_REQUEST_ZSTD,
   EVENT_FULLSCRAPE, /* TCP only */
   EVENT_FAILED,
   EVENT_BUCKET_LOCKED,