From 90cc56a14c0245ab8ba12ecb69f3fab2cfd2ba26 Mon Sep 17 00:00:00 2001
From: r4sas <r4sas@i2pmail.org>
Date: Tue, 23 Feb 2021 12:08:49 +0000
Subject: [PATCH] [fetch] add Last-Modified header usage

Signed-off-by: r4sas <r4sas@i2pmail.org>
---
 fetch.php                        | 22 +++++++++++++---------
 import.php                       | 29 +++++++++++------------------
 lib/utils.php                    | 10 ++++++++++
 database.sql => sql/database.sql |  1 +
 4 files changed, 35 insertions(+), 27 deletions(-)
 rename database.sql => sql/database.sql (96%)

diff --git a/fetch.php b/fetch.php
index 19fde21..aceda46 100644
--- a/fetch.php
+++ b/fetch.php
@@ -18,11 +18,14 @@ $aContext = array(
 );
 
 
-$STH = $pdo->query ("SELECT `name`, `url`, `etag` FROM `subscriptions` WHERE `active` = 1");
+$STH = $pdo->query ("SELECT `name`, `url`, `modified`, `etag` FROM `subscriptions` WHERE `active` = 1");
 $lists = $STH->fetchAll(PDO::FETCH_ASSOC);
 
 foreach ($lists as $list) {
-    echo "Processing " . $list['name'] . " subscription..." . PHP_EOL;
+    echo "Processing " . $list['name'] . " subscription...";
+
+    if (!empty($list['modified']))
+        $aContext['http']['header'] = 'If-Modified-Since: ' . $list['modified'] . '\r\n';
 
     if (!empty($list['etag']))
         $aContext['http']['header'] = 'If-None-Match: ' . $list['etag'] . '\r\n';
@@ -34,19 +37,20 @@ foreach ($lists as $list) {
         $f_meta = stream_get_meta_data($f);
 
         if (strpos($f_meta['wrapper_data'][0], "200") === false) {
+            echo " no changes." . PHP_EOL;
             continue;
         }
 
-        $etagHeader = array_filter($f_meta['wrapper_data'], function($el) {
-            return (strpos($el, "ETag") !== false);
-        });
+        $lastmod = $util->getResponseHeader("Last-Modified", $f_meta['wrapper_data']);
+        $etag    = $util->getResponseHeader("Etag", $f_meta['wrapper_data']);
 
-        if ($etagHeader) {
-            $etag = substr($etagHeader[array_keys($etagHeader)[0]], 6);
-            var_dump($etag);
-            $pdo->exec("UPDATE `subscriptions` SET `etag` = '" . $etag . "' WHERE `name` = '" . $list['name'] . "'");
+        if (!empty($lastmod) || !empty($etag)) {
+            $pdo->exec("UPDATE `subscriptions` SET " . (!empty($lastmod) ? ("`modified` = '" . $lastmod . "' ") : "") . (!empty($etag) ? ("`etag` = '" . $etag . "' ") : "") . "WHERE `name` = '" . $list['name'] . "'");
         }
 
+        // reset line
+        echo " fetching updated list." . PHP_EOL;
+
         while (($buffer = fgets($f, 4096)) !== false) {
             $domain = "";
             $record = $util->parseHostRecord($buffer);
diff --git a/import.php b/import.php
index e485597..f07fba7 100644
--- a/import.php
+++ b/import.php
@@ -16,30 +16,24 @@ if ($f) {
     $STH->bindParam('base64', $base64);
     $STH->bindParam('base32', $base32);
 
-    while (($buffer = fgets($f, 4096)) !== false)
-    {
+    while (($buffer = fgets($f, 4096)) !== false) {
         $domain = "";
         $record = $util->parseHostRecord($buffer);
 
-        if (!$util->isValidAddress($record['host'], $error))
-        {
+        if (!$util->isValidAddress($record['host'], $error)) {
             echo "Error while validating " . $record['host'] . ": " . $error . PHP_EOL;
             continue;
-        }
-        else
-        {
-            if($util->isPunycodeDomain($record['host']))
-            {
+
+        } else {
+            if($util->isPunycodeDomain($record['host'])) {
                 $domain = idn_to_utf8($record['host']);
-            }
-            else
-            {
+
+            } else {
                 $domain = $record['host'];
             }
         }
 
-        if (!$util->isValidBase64($record['b64']))
-        {
+        if (!$util->isValidBase64($record['b64'])) {
             echo "Error while validating " . $record['host'] . ": incorrect base64: " . $record['b64'] . PHP_EOL;
             continue;
         }
@@ -49,13 +43,12 @@ if ($f) {
 
         $STH->execute();
     }
-    if (!feof($f))
-    {
+
+    if (!feof($f)) {
         echo "Error: fgets() ended earlier than needed" . PHP_EOL;
     }
 
-    if (!$pdo->commit())
-    {
+    if (!$pdo->commit()) {
         echo "Error while saving records to database";
     }
 
diff --git a/lib/utils.php b/lib/utils.php
index 5c77652..bc4cd71 100644
--- a/lib/utils.php
+++ b/lib/utils.php
@@ -261,4 +261,14 @@ class Utils {
 
         return $record;
     }
+
+    public static function getResponseHeader($header, $response) {
+        foreach ($response as $key => $r) {
+            // Match the header name up to ':', compare lower case
+            if (stripos($r, $header . ':') === 0) {
+                list($headername, $headervalue) = explode(":", $r, 2);
+                return trim($headervalue);
+            }
+        }
+    }
 }
diff --git a/database.sql b/sql/database.sql
similarity index 96%
rename from database.sql
rename to sql/database.sql
index 2a2c071..a3c137e 100644
--- a/database.sql
+++ b/sql/database.sql
@@ -22,6 +22,7 @@ CREATE TABLE `hosts` (
 CREATE TABLE `subscriptions` (
   `name` varchar(67) NOT NULL,
   `url` varchar(256) NOT NULL,
+  `modified` char(30) NOT NULL DEFAULT '',
   `etag` varchar(64) NOT NULL DEFAULT '',
   `active` tinyint(1) NOT NULL DEFAULT 0
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;