regi2p/fetch.php

<?php
require __DIR__ . '/vendor/autoload.php';
require __DIR__ . '/config.php';

$pdo = (new App\DB($options))->pdo;
$util = new App\Utils;

$error = "";

$aContext = array(
    'http' => array(
        'method'          => 'GET',
        'proxy'           => $options['http_proxy'],
        'user_agent'      => 'MYOB/6.66 (AN/ON)',
        'request_fulluri' => true,
        'timeout'         => 120.0,
    ),
);


$STH = $pdo->query ("SELECT `name`, `url`, `modified`, `etag` FROM `subscriptions` WHERE `active` = 1");
$lists = $STH->fetchAll(PDO::FETCH_ASSOC);

foreach ($lists as $list) {
    echo "Processing " . $list['name'] . " subscription... ";

    if (!empty($list['modified']))
        $aContext['http']['header'] = 'If-Modified-Since: ' . $list['modified'] . '\r\n';

    if (!empty($list['etag']))
        $aContext['http']['header'] = 'If-None-Match: ' . $list['etag'] . '\r\n';

    $cxContext = stream_context_create($aContext);
    $f = fopen($list['url'], "r", false, $cxContext);

    if ($f) {
        $f_meta = stream_get_meta_data($f);

        if (strpos($f_meta['wrapper_data'][0], "200") === false) {
            echo "no changes (" . $f_meta['wrapper_data'][0]. ")" . PHP_EOL;
            continue;
        }

        $lastmod = $util->getResponseHeader("Last-Modified", $f_meta['wrapper_data']);
        $etag    = $util->getResponseHeader("Etag", $f_meta['wrapper_data']);

        if (!empty($lastmod) || !empty($etag)) {
            $pdo->exec("UPDATE `subscriptions` SET" . (!empty($lastmod) ? (" `modified` = '" . $lastmod . "'") : "") . (!empty($etag) ? ((!empty($lastmod) ? "," : "") . " `etag` = '" . $etag . "'") : "") . " WHERE `name` = '" . $list['name'] . "'");
        }

        // reset line
        echo "fetching updated list." . PHP_EOL;

        while (($buffer = fgets($f, 4096)) !== false) {
            if (substr($buffer, 0, 1) === "#")
                continue;

            $domain = "";
            $record = $util->parseHostRecord($buffer);

            if (!isset($record['host'])) {
                echo "Error while validating record " . $buffer . ": No host is found." . PHP_EOL;
                continue;

            } else if (!$util->isValidDomain($record['host'], $error)) {
                echo "Error while validating " . $record['host'] . ": " . $error . PHP_EOL;
                continue;

            } else {
                if ($util->isPunycodeDomain($record['host'])) {
                    $domain = idn_to_utf8($record['host']);

                } else {
                    $domain = $record['host'];
                }
            }

            if (!$util->isValidBase64($record['b64'])) {
                echo "Error while validating " . $record['host'] . ": invalid or unsupported base64 (len: " . strlen($record['b64']) . ")" . PHP_EOL;
                continue;
            }

            if ((isset($record['commands']) && isset($record['commands']['action']) && $record['commands']['action'] == "adddest" && isset($record['commands']['olddest'])) &&
                $pdo->query("SELECT COUNT(*) FROM `hosts` WHERE `host` = '" . $domain . "' AND `base64` =  '" . $record['commands']['olddest'] . "' LIMIT 1")->fetchColumn()) {

                $base32 = $util->b32from64($record['b64']);
                $pdo->exec("UPDATE `hosts` SET `base64` = '" . $record["b64"] . "', `base32` = '" . $base32 . "' WHERE `host` = '" . $domain . "'");
		echo "Processed " . $domain . " (adddest)" . PHP_EOL;
                continue;
            }

            if ((isset($record['commands']) && isset($record['commands']['action']) && $record['commands']['action'] == "addsubdomain" && isset($record['commands']['oldname']) && isset($record['commands']['olddest'])) &&
                ($pdo->query("SELECT COUNT(*) FROM `hosts` WHERE `host` = '" . $record['commands']['oldname'] . "' AND `base64` =  '" . $record['commands']['olddest'] . "' LIMIT 1")->fetchColumn() &&
                !$pdo->query("SELECT COUNT(*) FROM `hosts` WHERE `host` = '" . $domain . "'")->fetchColumn())) {

                $base32 = $util->b32from64($record['b64']);
                $pdo->exec("INSERT INTO `hosts` (`host`, `base64`, `base32`, `approved`) VALUES ('" . $domain . "', '" . $record["b64"] . "', '" . $base32 . "', 1)");
		echo "Processed " . $domain . " (addsubdomain)" . PHP_EOL;
                continue;
            }

            if(!$pdo->query("SELECT COUNT(*) FROM `hosts` WHERE `host` = '" . $domain . "' LIMIT 1")->fetchColumn()) {

                $base32 = $util->b32from64($record['b64']);
                $pdo->exec("INSERT INTO `hosts` (`host`, `base64`, `base32`, `approved`) VALUES ('" . $domain . "', '" . $record["b64"] . "', '" . $base32 . "', 1)");
		echo "Processed " . $domain . PHP_EOL;
                continue;
            }
        }

        if (!feof($f)) {
            echo "Error: fgets() ended earlier than needed." . PHP_EOL;
        }

        fclose($f);
    } else {
        echo "Empty response while fetching update from " . $list['name'].  "." . PHP_EOL;
    }
}

$pdo = null;