YGGverse/YGGo
1
0
Fork 0
mirror of https://github.com/YGGverse/YGGo.git synced 2025-01-24 13:34:25 +00:00
Code Issues Projects Releases Wiki Activity

add memcached support

Browse Source
This commit is contained in:
ghost 2023-07-27 17:53:36 +03:00
parent 0fb2e8a78c
commit 4a4394fb27
4 changed files with 91 additions and 41 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

2
README.md
View File

@ -30,6 +30,8 @@ php-gd
php-mbstring
php-zip
php-mysql
php-memcached
memcached
sphinxsearch
```

4
config/app.php.txt
View File

@ -86,6 +86,10 @@ define('DB_PASSWORD', '');
define('SPHINX_HOST', '127.0.0.1');
define('SPHINX_PORT', 9306);
// Memcached
define('MEMCACHED_HOST', '127.0.0.1');
define('MEMCACHED_PORT', 11211);
// Third-party connections (optional)
/*

106
library/mysql.php
View File

@ -3,13 +3,18 @@
class MySQL {
private PDO $_db;
private Memcached $_memcached;
public function __construct(string $host, int $port, string $database, string $username, string $password) {
public function __construct(string $host, int $port, string $database, string $username, string $password, Memcached $memcached = null) {
$this->_db = new PDO('mysql:dbname=' . $database . ';host=' . $host . ';port=' . $port . ';charset=utf8', $username, $password, [PDO::MYSQL_ATTR_INIT_COMMAND => 'SET NAMES utf8']);
$this->_db->setAttribute(PDO::ATTR_ERRMODE, PDO::ERRMODE_EXCEPTION);
$this->_db->setAttribute(PDO::ATTR_DEFAULT_FETCH_MODE, PDO::FETCH_OBJ);
$this->_db->setAttribute(PDO::ATTR_TIMEOUT, 600);
if ($memcached) {
$this->_memcached = $memcached;
}
}
// System
@ -86,57 +91,60 @@ class MySQL {
public function getTopHostPages() {
$query = $this->_db->query(" SELECT
if ($this->_memcached) {
`hostPageTarget`.`hostId`,
`hostPageTarget`.`hostPageId`,
`hostPageTarget`.`uri`,
if ($result = $this->_memcached->get('MySQL.getTopHostPages')) {
`hostTarget`.`scheme`,
`hostTarget`.`name`,
`hostTarget`.`port`,
return $result;
}
}
(
$query = $this->_db->query(" SELECT
SELECT COUNT(*) FROM `hostPage` AS `hostPageTotal`
`hostPageTarget`.`hostId`,
`hostPageTarget`.`hostPageId`,
`hostPageTarget`.`uri`,
WHERE `hostPageTotal`.`hostId` = `hostPageTarget`.`hostId`
`hostTarget`.`scheme`,
`hostTarget`.`name`,
`hostTarget`.`port`,
AND `hostPageTotal`.`httpCode` = 200
AND `hostPageTotal`.`timeBanned` IS NULL
AND `hostPageTotal`.`mime` IS NOT NULL
(
) AS `total`,
SELECT COUNT(*)
(
FROM `hostPageToHostPage`
JOIN `hostPage` AS `hostPageSource` ON (`hostPageSource`.`hostPageId` = `hostPageToHostPage`.`hostPageIdSource`)
SELECT COUNT(*)
WHERE `hostPageToHostPage`.`hostPageIdTarget` = `hostPageTarget`.`hostPageId`
AND `hostPageSource`.`hostId` <> `hostPageTarget`.`hostId`
FROM `hostPageToHostPage`
JOIN `hostPage` AS `hostPageSource` ON (`hostPageSource`.`hostPageId` = `hostPageToHostPage`.`hostPageIdSource`)
) AS `rank`
WHERE `hostPageToHostPage`.`hostPageIdTarget` = `hostPageTarget`.`hostPageId`
AND `hostPageSource`.`hostId` <> `hostPageTarget`.`hostId`
FROM `hostPage` AS `hostPageTarget`
JOIN `host` AS `hostTarget` ON (`hostTarget`.`hostId` = `hostPageTarget`.`hostId`)
) AS `rank`
WHERE `hostTarget`.`status` = '1'
AND `hostPageTarget`.`httpCode` = 200
AND `hostPageTarget`.`timeBanned` IS NULL
AND `hostPageTarget`.`mime` IS NOT NULL
FROM `hostPage` AS `hostPageTarget`
JOIN `host` AS `hostTarget` ON (`hostTarget`.`hostId` = `hostPageTarget`.`hostId`)
GROUP BY `hostPageTarget`.`hostPageId`
WHERE `hostTarget`.`status` = '1'
AND `hostPageTarget`.`httpCode` = 200
AND `hostPageTarget`.`timeBanned` IS NULL
AND `hostPageTarget`.`mime` IS NOT NULL
HAVING `rank` > 0
GROUP BY `hostPageTarget`.`hostPageId`
HAVING `rank` > 0
ORDER BY `rank` DESC
ORDER BY `rank` DESC
");
return $query->fetchAll();
$result = $query->fetchAll();
if ($this->_memcached) {
$this->_memcached->set('MySQL.getTopHostPages', $result, time() + 3600);
}
return $result;
}
public function getHosts() {
@ -225,6 +233,36 @@ class MySQL {
return $query->fetch()->total;
}
public function getTotalHostPagesIndexed(int $hostId) {
if ($this->_memcached) {
if ($result = $this->_memcached->get(sprintf('MySQL.getTotalHostPagesIndexed.%s', $hostId))) {
return $result;
}
}
$query = $this->_db->prepare('SELECT COUNT(*) AS `total` FROM `hostPage`
WHERE `hostId` = ?
AND `httpCode` = 200
AND `timeBanned` IS NULL
AND `mime` IS NOT NULL');
$query->execute([$hostId]);
$result = $query->fetch()->total;
if ($this->_memcached) {
$this->_memcached->set(sprintf('MySQL.getTotalHostPagesIndexed.%s', $hostId), $result, time() + 3600);
}
return $result;
}
/* not in use
public function getTotalPagesByHttpCode(mixed $httpCode) {

20
public/top.php
View File

@ -9,8 +9,12 @@ require_once(__DIR__ . '/../library/sphinxql.php');
// Connect Sphinx search server
$sphinx = new SphinxQL(SPHINX_HOST, SPHINX_PORT);
// Connect Memcached
$memcached = new Memcached();
$memcached->addServer(MEMCACHED_HOST, MEMCACHED_PORT);
// Connect database
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD);
$db = new MySQL(DB_HOST, DB_PORT, DB_NAME, DB_USERNAME, DB_PASSWORD, $memcached);
// Define page basics
$totalPages = $sphinx->getHostPagesTotal();
@ -20,8 +24,6 @@ $placeholder = Filter::plural($totalPages, [sprintf(_('Over %s page or enter the
sprintf(_('Over %s pages or enter the new one...'), $totalPages),
]);
?>
<!DOCTYPE html>
@ -240,7 +242,9 @@ $placeholder = Filter::plural($totalPages, [sprintf(_('Over %s page or enter the
</tr>
<?php foreach ($topHostPages as $i => $topHostPage) { ?>
<tr>
<td><?php echo $i + 1 ?></td>
<td>
<?php echo $i + 1 ?>
</td>
<td>
<?php if ($hostPageDescription = $db->getLastPageDescription($topHostPage->hostPageId)) { ?>
<?php if (!empty($hostPageDescription->title)) { ?>
@ -259,10 +263,12 @@ $placeholder = Filter::plural($totalPages, [sprintf(_('Over %s page or enter the
</a>
</td>
<td>
<?php $totalHostPages = $db->getTotalHostPages($topHostPage->hostId) ?>
<?php echo $totalHostPages . ($totalHostPages >= CRAWL_HOST_DEFAULT_PAGES_LIMIT ? '+' : false) ?>
<?php $totalHostPagesIndexed = $db->getTotalHostPagesIndexed($topHostPage->hostId) ?>
<?php echo $totalHostPagesIndexed . ($totalHostPagesIndexed >= CRAWL_HOST_DEFAULT_PAGES_LIMIT ? '+' : false) ?>
</td>
<td>
<?php echo $topHostPage->rank ?>
</td>
<td><?php echo $topHostPage->rank ?></td>
<td>
<a href="<?php echo WEBSITE_DOMAIN; ?>/explore.php?hp=<?php echo $topHostPage->hostPageId ?>">
<?php echo _('explore'); ?>