YGGverse/Yo
1
0
Fork 0
mirror of https://github.com/YGGverse/Yo.git synced 2025-03-13 05:41:18 +00:00
Code Issues Projects Releases Wiki Activity

add stripos url rules for crawl snaps

Browse Source
This commit is contained in:
ghost 2023-12-02 22:15:44 +02:00
parent 0e06ff3c0f
commit 35ad144a9e
2 changed files with 96 additions and 32 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

22
example/config.json
View File

@ -192,13 +192,15 @@
"directory":"storage/tmp/snap" "directory":"storage/tmp/snap"
}, },
"local":{ "local":{
"enabled":true, "enabled":false,
"directory":"storage/snap", "directory":"storage/snap",
"size": "size":
{ {
"max":10000024 "max":10000024
}, },
"mime": "mime":
{
"stripos":
[ [
"application/xhtml+xml", "application/xhtml+xml",
"application/javascript", "application/javascript",
@ -211,6 +213,14 @@
"image/ico" "image/ico"
] ]
}, },
"url":
{
"stripos":
[
"http"
]
}
},
"remote": "remote":
{ {
"ftp": "ftp":
@ -237,6 +247,8 @@
"max":10000024 "max":10000024
}, },
"mime": "mime":
{
"stripos":
[ [
"application/xhtml+xml", "application/xhtml+xml",
"application/javascript", "application/javascript",
@ -248,6 +260,14 @@
"image/gif", "image/gif",
"image/ico" "image/ico"
] ]
},
"url":
{
"stripos":
[
"http"
]
}
} }
] ]
} }

60
src/cli/document/crawl.php
View File

@ -479,10 +479,10 @@ foreach($search->get() as $document)
// Copy to local storage on enabled // Copy to local storage on enabled
if ($config->snap->storage->local->enabled) if ($config->snap->storage->local->enabled)
{ {
// Check for mime allowed
$allowed = false; $allowed = false;
// Check for mime allowed foreach ($config->snap->storage->local->mime->stripos as $whitelist)
foreach ($config->snap->storage->local->mime as $whitelist)
{ {
if (false !== stripos($mime, $whitelist)) if (false !== stripos($mime, $whitelist))
{ {
@ -491,10 +491,30 @@ foreach($search->get() as $document)
} }
} }
// Check size limits // Check for url allowed
if ($size > $config->snap->storage->local->size->max) if ($allowed)
{ {
$allowed = false; $allowed = false;
foreach ($config->snap->storage->local->url->stripos as $whitelist)
{
if (false !== stripos($document->get('url'), $whitelist))
{
$allowed = true;
break;
}
}
// Check size limits
if ($allowed)
{
$allowed = false;
if ($size <= $config->snap->storage->local->size->max)
{
$allowed = true;
}
}
} }
// Copy snap to the permanent storage // Copy snap to the permanent storage
@ -558,10 +578,10 @@ foreach($search->get() as $document)
continue; continue;
} }
// Check for mime allowed
$allowed = false; $allowed = false;
// Check for mime allowed foreach ($ftp->mime->stripos as $whitelist)
foreach ($ftp->mime as $whitelist)
{ {
if (false !== stripos($mime, $whitelist)) if (false !== stripos($mime, $whitelist))
{ {
@ -570,10 +590,34 @@ foreach($search->get() as $document)
} }
} }
// Check size limits if (!$allowed)
if ($size > $ftp->size->max)
{ {
continue;
}
// Check for url allowed
$allowed = false; $allowed = false;
foreach ($ftp->url->stripos as $whitelist)
{
if (false !== stripos($document->get('url'), $whitelist))
{
$allowed = true;
break;
}
}
if (!$allowed)
{
continue;
}
// Check size limits
$allowed = false;
if ($size <= $ftp->size->max)
{
$allowed = true;
} }
if (!$allowed) if (!$allowed)