[fetch] add Last-Modified header usage

Signed-off-by: r4sas <r4sas@i2pmail.org>

fetch.php

@@ -18,11 +18,14 @@ $aContext = array(
 );
 
 
-$STH = $pdo->query ("SELECT `name`, `url`, `etag` FROM `subscriptions` WHERE `active` = 1");
+$STH = $pdo->query ("SELECT `name`, `url`, `modified`, `etag` FROM `subscriptions` WHERE `active` = 1");
 $lists = $STH->fetchAll(PDO::FETCH_ASSOC);
 
 foreach ($lists as $list) {
-    echo "Processing " . $list['name'] . " subscription..." . PHP_EOL;
+    echo "Processing " . $list['name'] . " subscription...";
+
+    if (!empty($list['modified']))
+        $aContext['http']['header'] = 'If-Modified-Since: ' . $list['modified'] . '\r\n';
 
     if (!empty($list['etag']))
         $aContext['http']['header'] = 'If-None-Match: ' . $list['etag'] . '\r\n';
@@ -34,19 +37,20 @@ foreach ($lists as $list) {
         $f_meta = stream_get_meta_data($f);
 
         if (strpos($f_meta['wrapper_data'][0], "200") === false) {
+            echo " no changes." . PHP_EOL;
             continue;
         }
 
-        $etagHeader = array_filter($f_meta['wrapper_data'], function($el) {
+        $lastmod = $util->getResponseHeader("Last-Modified", $f_meta['wrapper_data']);
-            return (strpos($el, "ETag") !== false);
+        $etag    = $util->getResponseHeader("Etag", $f_meta['wrapper_data']);
-        });
 
-        if ($etagHeader) {
+        if (!empty($lastmod) || !empty($etag)) {
-            $etag = substr($etagHeader[array_keys($etagHeader)[0]], 6);
+            $pdo->exec("UPDATE `subscriptions` SET " . (!empty($lastmod) ? ("`modified` = '" . $lastmod . "' ") : "") . (!empty($etag) ? ("`etag` = '" . $etag . "' ") : "") . "WHERE `name` = '" . $list['name'] . "'");
-            var_dump($etag);
-            $pdo->exec("UPDATE `subscriptions` SET `etag` = '" . $etag . "' WHERE `name` = '" . $list['name'] . "'");
         }
 
+        // reset line
+        echo " fetching updated list." . PHP_EOL;
+
         while (($buffer = fgets($f, 4096)) !== false) {
             $domain = "";
             $record = $util->parseHostRecord($buffer);

import.php

@@ -16,30 +16,24 @@ if ($f) {
     $STH->bindParam('base64', $base64);
     $STH->bindParam('base32', $base32);
 
-    while (($buffer = fgets($f, 4096)) !== false)
+    while (($buffer = fgets($f, 4096)) !== false) {
-    {
         $domain = "";
         $record = $util->parseHostRecord($buffer);
 
-        if (!$util->isValidAddress($record['host'], $error))
+        if (!$util->isValidAddress($record['host'], $error)) {
-        {
             echo "Error while validating " . $record['host'] . ": " . $error . PHP_EOL;
             continue;
-        }
+
-        else
+        } else {
-        {
+            if($util->isPunycodeDomain($record['host'])) {
-            if($util->isPunycodeDomain($record['host']))
-            {
                 $domain = idn_to_utf8($record['host']);
-            }
+
-            else
+            } else {
-            {
                 $domain = $record['host'];
             }
         }
 
-        if (!$util->isValidBase64($record['b64']))
+        if (!$util->isValidBase64($record['b64'])) {
-        {
             echo "Error while validating " . $record['host'] . ": incorrect base64: " . $record['b64'] . PHP_EOL;
             continue;
         }
@@ -49,13 +43,12 @@ if ($f) {
 
         $STH->execute();
     }
-    if (!feof($f))
+
-    {
+    if (!feof($f)) {
         echo "Error: fgets() ended earlier than needed" . PHP_EOL;
     }
 
-    if (!$pdo->commit())
+    if (!$pdo->commit()) {
-    {
         echo "Error while saving records to database";
     }
 

lib/utils.php

@@ -261,4 +261,14 @@ class Utils {
 
         return $record;
     }
+
+    public static function getResponseHeader($header, $response) {
+        foreach ($response as $key => $r) {
+            // Match the header name up to ':', compare lower case
+            if (stripos($r, $header . ':') === 0) {
+                list($headername, $headervalue) = explode(":", $r, 2);
+                return trim($headervalue);
+            }
+        }
+    }
 }

sql/database.sql

@@ -22,6 +22,7 @@ CREATE TABLE `hosts` (
 CREATE TABLE `subscriptions` (
   `name` varchar(67) NOT NULL,
   `url` varchar(256) NOT NULL,
+  `modified` char(30) NOT NULL DEFAULT '',
   `etag` varchar(64) NOT NULL DEFAULT '',
   `active` tinyint(1) NOT NULL DEFAULT 0
 ) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;