PurpleI2P/regi2p
1
0
Fork 0
mirror of https://github.com/PurpleI2P/regi2p.git synced 2025-02-02 06:54:16 +00:00
Code Issues Releases Activity

[fetch] add Last-Modified header usage

Browse Source 
Signed-off-by: r4sas <r4sas@i2pmail.org>
This commit is contained in:
R4SAS 2021-02-23 12:08:49 +00:00
parent 80bb08fbd2
commit 90cc56a14c
Signed by: r4sas
GPG Key ID: 66F6C87B98EBCFE2
4 changed files with 35 additions and 27 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
fetch.php
View File

@ -18,11 +18,14 @@ $aContext = array(
);
$STH = $pdo->query ("SELECT `name`, `url`, `etag` FROM `subscriptions` WHERE `active` = 1");
$STH = $pdo->query ("SELECT `name`, `url`, `modified`, `etag` FROM `subscriptions` WHERE `active` = 1");
$lists = $STH->fetchAll(PDO::FETCH_ASSOC);
foreach ($lists as $list) {
echo "Processing " . $list['name'] . " subscription..." . PHP_EOL;
echo "Processing " . $list['name'] . " subscription...";
if (!empty($list['modified']))
$aContext['http']['header'] = 'If-Modified-Since: ' . $list['modified'] . '\r\n';
if (!empty($list['etag']))
$aContext['http']['header'] = 'If-None-Match: ' . $list['etag'] . '\r\n';
@ -34,19 +37,20 @@ foreach ($lists as $list) {
$f_meta = stream_get_meta_data($f);
if (strpos($f_meta['wrapper_data'][0], "200") === false) {
echo " no changes." . PHP_EOL;
continue;
}
$etagHeader = array_filter($f_meta['wrapper_data'], function($el) {
return (strpos($el, "ETag") !== false);
});
$lastmod = $util->getResponseHeader("Last-Modified", $f_meta['wrapper_data']);
$etag = $util->getResponseHeader("Etag", $f_meta['wrapper_data']);
if ($etagHeader) {
$etag = substr($etagHeader[array_keys($etagHeader)[0]], 6);
var_dump($etag);
$pdo->exec("UPDATE `subscriptions` SET `etag` = '" . $etag . "' WHERE `name` = '" . $list['name'] . "'");
if (!empty($lastmod) || !empty($etag)) {
$pdo->exec("UPDATE `subscriptions` SET " . (!empty($lastmod) ? ("`modified` = '" . $lastmod . "' ") : "") . (!empty($etag) ? ("`etag` = '" . $etag . "' ") : "") . "WHERE `name` = '" . $list['name'] . "'");
}
// reset line
echo " fetching updated list." . PHP_EOL;
while (($buffer = fgets($f, 4096)) !== false) {
$domain = "";
$record = $util->parseHostRecord($buffer);

29
import.php
View File

@ -16,30 +16,24 @@ if ($f) {
$STH->bindParam('base64', $base64);
$STH->bindParam('base32', $base32);
while (($buffer = fgets($f, 4096)) !== false)
{
while (($buffer = fgets($f, 4096)) !== false) {
$domain = "";
$record = $util->parseHostRecord($buffer);
if (!$util->isValidAddress($record['host'], $error))
{
if (!$util->isValidAddress($record['host'], $error)) {
echo "Error while validating " . $record['host'] . ": " . $error . PHP_EOL;
continue;
}
else
{
if($util->isPunycodeDomain($record['host']))
{
} else {
if($util->isPunycodeDomain($record['host'])) {
$domain = idn_to_utf8($record['host']);
}
else
{
} else {
$domain = $record['host'];
}
}
if (!$util->isValidBase64($record['b64']))
{
if (!$util->isValidBase64($record['b64'])) {
echo "Error while validating " . $record['host'] . ": incorrect base64: " . $record['b64'] . PHP_EOL;
continue;
}
@ -49,13 +43,12 @@ if ($f) {
$STH->execute();
}
if (!feof($f))
{
if (!feof($f)) {
echo "Error: fgets() ended earlier than needed" . PHP_EOL;
}
if (!$pdo->commit())
{
if (!$pdo->commit()) {
echo "Error while saving records to database";
}

10
lib/utils.php
View File

@ -261,4 +261,14 @@ class Utils {
return $record;
}
public static function getResponseHeader($header, $response) {
foreach ($response as $key => $r) {
// Match the header name up to ':', compare lower case
if (stripos($r, $header . ':') === 0) {
list($headername, $headervalue) = explode(":", $r, 2);
return trim($headervalue);
}
}
}
}

1
database.sql → sql/database.sql
View File

@ -22,6 +22,7 @@ CREATE TABLE `hosts` (
CREATE TABLE `subscriptions` (
`name` varchar(67) NOT NULL,
`url` varchar(256) NOT NULL,
`modified` char(30) NOT NULL DEFAULT '',
`etag` varchar(64) NOT NULL DEFAULT '',
`active` tinyint(1) NOT NULL DEFAULT 0
) ENGINE=InnoDB DEFAULT CHARSET=utf8mb4;