mirror of
https://github.com/Karaka-Management/oms-OnlineResourceWatcher.git
synced 2026-02-16 16:08:40 +00:00
continue implementation
This commit is contained in:
Dennis Eichhorn
parent
d3057e9f0b
commit
b286b5df03
|
|
@ -49,6 +49,10 @@ use phpOMS\Utils\StringUtils;
|
||||||
*/
|
*/
|
||||||
final class ApiController extends Controller
|
final class ApiController extends Controller
|
||||||
{
|
{
|
||||||
|
public const TEXT_RENDERABLE = ['md', 'txt', 'doc', 'docx', 'pdf', 'xls', 'xlsx'];
|
||||||
|
|
||||||
|
public const IMG_RENDERABLE = ['png', 'jpg', 'jpeg', 'gif', 'bmp', 'svg'];
|
||||||
|
|
||||||
/**
|
/**
|
||||||
* Api method to create resource
|
* Api method to create resource
|
||||||
*
|
*
|
||||||
|
|
@ -310,12 +314,11 @@ final class ApiController extends Controller
|
||||||
|
|
||||||
// Check downloaded resources
|
// Check downloaded resources
|
||||||
$totalCount = \count($toCheck);
|
$totalCount = \count($toCheck);
|
||||||
$maxLoops = (int) \min(60 * 10, $totalCount * 10 / 4);
|
$maxLoops = (int) \min(60 * 10, $totalCount * 10 / 4); // At most run 600 times or 2.5 times the resource count
|
||||||
$startTime = \time();
|
$startTime = \time();
|
||||||
$minTime = $startTime + \min(10 * $totalCount, 60 * 5); // At least run 10 seconds per element
|
$minTime = $startTime + ((int) \min(10 * $totalCount, 60 * 5)); // At least run 10 seconds per element or 5 minutes
|
||||||
$maxTime = $startTime + \min(60 * $totalCount, 60 * 60 * 3); // At most run 60 seconds per element
|
$maxTime = $startTime + ((int) \min(60 * $totalCount, 60 * 60 * 3)); // At most run 60 seconds per element or 3 hours
|
||||||
|
|
||||||
$baseLen = \strlen($basePath . '/temp');
|
|
||||||
while (!empty($toCheck)) {
|
while (!empty($toCheck)) {
|
||||||
$time = \time();
|
$time = \time();
|
||||||
|
|
||||||
|
|
@ -325,7 +328,9 @@ final class ApiController extends Controller
|
||||||
/** @var Resource $resource */
|
/** @var Resource $resource */
|
||||||
$resource = $check['resource'];
|
$resource = $check['resource'];
|
||||||
|
|
||||||
// too many tries
|
// Too many tries
|
||||||
|
// 1. Too many iterations and at least the min execution time is reached
|
||||||
|
// 2. Execution takes longer than max time
|
||||||
if (($check['loop'] > $maxLoops && $time > $minTime) || $time > $maxTime) {
|
if (($check['loop'] > $maxLoops && $time > $minTime) || $time > $maxTime) {
|
||||||
$report = new Report();
|
$report = new Report();
|
||||||
$report->resource = $resource->id;
|
$report->resource = $resource->id;
|
||||||
|
|
@ -339,6 +344,8 @@ final class ApiController extends Controller
|
||||||
|
|
||||||
unset($toCheck[$index]);
|
unset($toCheck[$index]);
|
||||||
|
|
||||||
|
$this->app->logger->warning('Resource "' . $resource->id . ': ' . $resource->uri . '" took too long to download.');
|
||||||
|
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -349,9 +356,6 @@ final class ApiController extends Controller
|
||||||
continue;
|
continue;
|
||||||
}
|
}
|
||||||
|
|
||||||
$end = \stripos($path, '/', $baseLen + 1);
|
|
||||||
$id = (int) \substr($path, $baseLen + 1, $end - $baseLen - 1);
|
|
||||||
|
|
||||||
// new resource
|
// new resource
|
||||||
$filesNew = \scandir($path);
|
$filesNew = \scandir($path);
|
||||||
if ($filesNew === false) {
|
if ($filesNew === false) {
|
||||||
|
|
@ -375,20 +379,13 @@ final class ApiController extends Controller
|
||||||
$toCheck[$index]['handled'] = true;
|
$toCheck[$index]['handled'] = true;
|
||||||
} else {
|
} else {
|
||||||
foreach ($filesNew as $file) {
|
foreach ($filesNew as $file) {
|
||||||
if (StringUtils::endsWith($file, '.png')
|
if ($file === '..' || $file = '.') {
|
||||||
|| StringUtils::endsWith($file, '.jpg')
|
continue;
|
||||||
|| StringUtils::endsWith($file, '.jpeg')
|
}
|
||||||
|| StringUtils::endsWith($file, '.gif')
|
|
||||||
|| StringUtils::endsWith($file, '.pdf')
|
$extension = ($pos = \strrpos($file, '.')) !== false ? \substr($file, $pos + 1) : '';
|
||||||
|| StringUtils::endsWith($file, '.doc')
|
if (StringUtils::endsWith($file, '.' . $extension)) {
|
||||||
|| StringUtils::endsWith($file, '.docx')
|
|
||||||
|| StringUtils::endsWith($file, '.xls')
|
|
||||||
|| StringUtils::endsWith($file, '.xlsx')
|
|
||||||
|| StringUtils::endsWith($file, '.md')
|
|
||||||
|| StringUtils::endsWith($file, '.txt')
|
|
||||||
) {
|
|
||||||
$fileName = $file;
|
$fileName = $file;
|
||||||
$extension = \substr($file, \strripos($file, '.') + 1);
|
|
||||||
$toCheck[$index]['handled'] = true;
|
$toCheck[$index]['handled'] = true;
|
||||||
|
|
||||||
break;
|
break;
|
||||||
|
|
@ -397,38 +394,40 @@ final class ApiController extends Controller
|
||||||
}
|
}
|
||||||
|
|
||||||
// Is new resource
|
// Is new resource
|
||||||
if (!\is_dir($basePath . '/' . $id)) {
|
if (!\is_dir($basePath . '/' . $resource->id)) {
|
||||||
$report = new Report();
|
$report = new Report();
|
||||||
$report->resource = $resource->id;
|
$report->resource = $resource->id;
|
||||||
$report->versionPath = (string) $check['timestamp'];
|
$report->versionPath = (string) $check['timestamp'];
|
||||||
|
|
||||||
$hash = '';
|
$hash = '';
|
||||||
if (!empty($fileName)) {
|
if (empty($fileName)) {
|
||||||
|
$report->status = ReportStatus::DOWNLOAD_ERROR;
|
||||||
|
} else {
|
||||||
$report->status = ReportStatus::ADDED;
|
$report->status = ReportStatus::ADDED;
|
||||||
$hash = \md5_file($path . '/' . $fileName);
|
$hash = \md5_file($path . '/' . $fileName);
|
||||||
} else {
|
|
||||||
$report->status = ReportStatus::DOWNLOAD_ERROR;
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
|
$this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
|
||||||
$old = clone $resource;
|
$old = clone $resource;
|
||||||
$resource->path = (string) $resource->id;
|
$resource->path = $resource->id === 0 ? '' : (string) $resource->id;
|
||||||
$resource->lastVersionPath = (string) $check['timestamp'];
|
$resource->lastVersionPath = (string) $check['timestamp'];
|
||||||
$resource->lastVersionDate = $report->createdAt;
|
$resource->lastVersionDate = $report->createdAt;
|
||||||
$resource->hash = $hash == false ? '' : $hash;
|
$resource->hash = $hash == false ? '' : $hash;
|
||||||
$resource->checkedAt = $report->createdAt;
|
$resource->checkedAt = $report->createdAt;
|
||||||
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
|
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
|
||||||
|
|
||||||
Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
|
Directory::copy($path, $basePath . '/' . $resource->id . '/' . $check['timestamp']);
|
||||||
unset($toCheck[$index]);
|
unset($toCheck[$index]);
|
||||||
|
|
||||||
if ($extension === 'htm') {
|
if ($extension === 'htm') {
|
||||||
try {
|
try {
|
||||||
SystemUtils::runProc(
|
SystemUtils::runProc(
|
||||||
OperatingSystem::getSystem() === SystemType::WIN ? 'wkhtmltoimage.exe' : 'wkhtmltoimage',
|
OperatingSystem::getSystem() === SystemType::WIN ? 'wkhtmltoimage.exe' : 'wkhtmltoimage',
|
||||||
\escapeshellarg($resource->uri) . ' ' . \escapeshellarg($basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg'),
|
\escapeshellarg($resource->uri) . ' ' . \escapeshellarg($basePath . '/' . $resource->id . '/' . $check['timestamp'] . '/index.jpg'),
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
|
|
||||||
|
\var_dump('wkhtmltoimage ' . \escapeshellarg($resource->uri) . ' ' . \escapeshellarg($basePath . '/' . $resource->id . '/' . $check['timestamp'] . '/index.jpg'));
|
||||||
} catch (\Throwable $t) {
|
} catch (\Throwable $t) {
|
||||||
$this->app->logger->error($t->getMessage());
|
$this->app->logger->error($t->getMessage());
|
||||||
}
|
}
|
||||||
|
|
@ -438,7 +437,7 @@ final class ApiController extends Controller
|
||||||
}
|
}
|
||||||
|
|
||||||
// existing resource
|
// existing resource
|
||||||
$resourcePaths = \scandir($basePath . '/' . $id);
|
$resourcePaths = \scandir($basePath . '/' . $resource->id);
|
||||||
if ($resourcePaths === false) {
|
if ($resourcePaths === false) {
|
||||||
$resourcePaths = [];
|
$resourcePaths = [];
|
||||||
}
|
}
|
||||||
|
|
@ -448,13 +447,9 @@ final class ApiController extends Controller
|
||||||
$lastVersionTimestamp = \end($resourcePaths);
|
$lastVersionTimestamp = \end($resourcePaths);
|
||||||
if ($lastVersionTimestamp === '.' || $lastVersionTimestamp === '..') {
|
if ($lastVersionTimestamp === '.' || $lastVersionTimestamp === '..') {
|
||||||
$lastVersionTimestamp = \reset($resourcePaths);
|
$lastVersionTimestamp = \reset($resourcePaths);
|
||||||
|
|
||||||
if ($lastVersionTimestamp === '.' || $lastVersionTimestamp === '..') {
|
|
||||||
Directory::delete($basePath . '/' . $id);
|
|
||||||
}
|
|
||||||
}
|
}
|
||||||
|
|
||||||
$lastVersionPath = $basePath . '/' . $id . '/' . $lastVersionTimestamp;
|
$lastVersionPath = $basePath . '/' . $resource->id . '/' . $lastVersionTimestamp;
|
||||||
$oldPath = $lastVersionPath . '/' . $fileName;
|
$oldPath = $lastVersionPath . '/' . $fileName;
|
||||||
$newPath = $path . '/' . $fileName;
|
$newPath = $path . '/' . $fileName;
|
||||||
|
|
||||||
|
|
@ -475,10 +470,11 @@ final class ApiController extends Controller
|
||||||
|
|
||||||
$difference = 0;
|
$difference = 0;
|
||||||
if ($hasDifferentHash) {
|
if ($hasDifferentHash) {
|
||||||
if (\in_array($extension, ['md', 'txt', 'doc', 'docx', 'pdf', 'xls', 'xlsx'])) {
|
if (\in_array($extension, self::TEXT_RENDERABLE)) {
|
||||||
$contentOld = \Modules\Media\Controller\ApiController::loadFileContent($oldPath, $extension);
|
$contentOld = \Modules\Media\Controller\ApiController::loadFileContent($oldPath, $extension);
|
||||||
$contentNew = \Modules\Media\Controller\ApiController::loadFileContent($newPath, $extension);
|
$contentNew = \Modules\Media\Controller\ApiController::loadFileContent($newPath, $extension);
|
||||||
|
|
||||||
|
// Calculate difference index
|
||||||
$difference = \levenshtein($contentOld, $contentNew);
|
$difference = \levenshtein($contentOld, $contentNew);
|
||||||
|
|
||||||
// Handle xpath
|
// Handle xpath
|
||||||
|
|
@ -512,10 +508,15 @@ final class ApiController extends Controller
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
// Calculate difference index
|
||||||
$difference = \levenshtein($subcontentOld, $subcontentNew);
|
$difference = \levenshtein($subcontentOld, $subcontentNew);
|
||||||
}
|
}
|
||||||
} elseif (\in_array($extension, ['png', 'jpg', 'jpeg', 'gif', 'bmp', 'svg'])) {
|
} elseif (\in_array($extension, self::IMG_RENDERABLE)) {
|
||||||
|
// Difference index is always 0/1. Comparing pixels is too slow and doesn't add much value
|
||||||
$difference = 1; //ImageUtils::difference($oldPath, $newPath, $path . '/_' . \basename($newPath), 0); // too slow
|
$difference = 1; //ImageUtils::difference($oldPath, $newPath, $path . '/_' . \basename($newPath), 0); // too slow
|
||||||
|
} else {
|
||||||
|
// All other files always have a difference index of 0/1
|
||||||
|
$difference = 1;
|
||||||
}
|
}
|
||||||
}
|
}
|
||||||
|
|
||||||
|
|
@ -536,14 +537,14 @@ final class ApiController extends Controller
|
||||||
|
|
||||||
$changed[] = $report;
|
$changed[] = $report;
|
||||||
|
|
||||||
Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
|
Directory::copy($path, $basePath . '/' . $resource->id . '/' . $check['timestamp']);
|
||||||
|
|
||||||
// If is htm/html create image
|
// If is htm/html create image
|
||||||
if ($extension === 'htm') {
|
if ($extension === 'htm') {
|
||||||
try {
|
try {
|
||||||
SystemUtils::runProc(
|
SystemUtils::runProc(
|
||||||
OperatingSystem::getSystem() === SystemType::WIN ? 'wkhtmltoimage.exe' : 'wkhtmltoimage',
|
OperatingSystem::getSystem() === SystemType::WIN ? 'wkhtmltoimage.exe' : 'wkhtmltoimage',
|
||||||
\escapeshellarg($resource->uri) . ' ' . \escapeshellarg($basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg'),
|
\escapeshellarg($resource->uri) . ' ' . \escapeshellarg($basePath . '/' . $resource->id . '/' . $check['timestamp'] . '/index.jpg'),
|
||||||
true
|
true
|
||||||
);
|
);
|
||||||
} catch (\Throwable $t) {
|
} catch (\Throwable $t) {
|
||||||
|
|
@ -619,17 +620,17 @@ final class ApiController extends Controller
|
||||||
$resource->checkedAt = $report->createdAt;
|
$resource->checkedAt = $report->createdAt;
|
||||||
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
|
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
|
||||||
|
|
||||||
// Directory::delete($basePath . '/temp/' . $id);
|
// Directory::delete($basePath . '/temp/' . $resource->id);
|
||||||
|
|
||||||
// @todo: delete older history depending on plan
|
// @todo: delete older history depending on plan
|
||||||
|
|
||||||
unset($toCheck[$index]);
|
unset($toCheck[$index]);
|
||||||
}
|
}
|
||||||
|
|
||||||
\usleep(100000); // 100ms
|
\usleep(100000); // 100ms delay to give the tools time to download between steps
|
||||||
}
|
}
|
||||||
|
|
||||||
// make sure that no wkhtmltoimage processes are running
|
// Make sure that no wkhtmltoimage processes are running
|
||||||
if (OperatingSystem::getSystem() === SystemType::LINUX) {
|
if (OperatingSystem::getSystem() === SystemType::LINUX) {
|
||||||
SystemUtils::runProc('pkill', '-f wkhtmltoimage', true);
|
SystemUtils::runProc('pkill', '-f wkhtmltoimage', true);
|
||||||
} else {
|
} else {
|
||||||
|
|
|
||||||
|
|
@ -12,6 +12,7 @@
|
||||||
*/
|
*/
|
||||||
declare(strict_types=1);
|
declare(strict_types=1);
|
||||||
|
|
||||||
|
use Modules\OnlineResourceWatcher\Models\ReportStatus;
|
||||||
use phpOMS\Uri\UriFactory;
|
use phpOMS\Uri\UriFactory;
|
||||||
|
|
||||||
/** @var \Modules\OnlineResourceWatcher\Models\Resource */
|
/** @var \Modules\OnlineResourceWatcher\Models\Resource */
|
||||||
|
|
@ -19,7 +20,7 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
|
||||||
$reports = $resource->reports;
|
$reports = $resource->reports;
|
||||||
?>
|
?>
|
||||||
<div class="row">
|
<div class="row">
|
||||||
<div class="col-xs-8">
|
<div class="col-xs-12 col-sm-8">
|
||||||
<div class="portlet">
|
<div class="portlet">
|
||||||
<form id="iResource" action="<?= UriFactory::build('{/api}resource'); ?>" method="post">
|
<form id="iResource" action="<?= UriFactory::build('{/api}resource'); ?>" method="post">
|
||||||
<div class="portlet-head"><?= $this->getHtml('Resource'); ?></div>
|
<div class="portlet-head"><?= $this->getHtml('Resource'); ?></div>
|
||||||
|
|
@ -56,14 +57,15 @@ $reports = $resource->reports;
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
||||||
<div class="col-xs-4">
|
<div class="col-xs-12 col-sm-4">
|
||||||
<div class="portlet">
|
<div class="portlet">
|
||||||
<div class="portlet-head"><?= $this->getHtml('History'); ?></div>
|
<div class="portlet-head"><?= $this->getHtml('History'); ?></div>
|
||||||
|
<div class="slider">
|
||||||
<table class="default">
|
<table class="default">
|
||||||
<thead>
|
<thead>
|
||||||
<tr>
|
<tr>
|
||||||
<td><?= $this->getHtml('Date'); ?>
|
<td><?= $this->getHtml('Date'); ?>
|
||||||
<td><?= $this->getHtml('Status'); ?>
|
<td class="wf-100"><?= $this->getHtml('Status'); ?>
|
||||||
<tbody>
|
<tbody>
|
||||||
<?php foreach ($reports as $report) : ?>
|
<?php foreach ($reports as $report) : ?>
|
||||||
<tr>
|
<tr>
|
||||||
|
|
@ -71,6 +73,7 @@ $reports = $resource->reports;
|
||||||
<td><?= $this->getHtml('rstatus-' . $report->status); ?>
|
<td><?= $this->getHtml('rstatus-' . $report->status); ?>
|
||||||
<?php endforeach; ?>
|
<?php endforeach; ?>
|
||||||
</table>
|
</table>
|
||||||
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
@ -119,12 +122,14 @@ $reports = $resource->reports;
|
||||||
}
|
}
|
||||||
?>
|
?>
|
||||||
|
|
||||||
<?php if ($type === 'img') : ?>
|
<?php if ($report->status !== ReportStatus::DOWNLOAD_ERROR) : ?>
|
||||||
<img src="<?= UriFactory::build($webPath); ?>" alt="<?= $this->printHtml($resource->title); ?>">
|
<?php if ($type === 'img') : ?>
|
||||||
<?php elseif ($type === 'pdf') : ?>
|
<img src="<?= UriFactory::build($webPath); ?>" alt="<?= $this->printHtml($resource->title); ?>">
|
||||||
<iframe class="col-simple" id="iRenderFrame" src="Resources/mozilla/Pdf/web/viewer.html?file=<?= \urlencode($webPath); ?>" loading="lazy" allowfullscreen></iframe>
|
<?php elseif ($type === 'pdf') : ?>
|
||||||
<?php else : ?>
|
<iframe class="col-simple" id="iRenderFrame" src="Resources/mozilla/Pdf/web/viewer.html?file=<?= \urlencode($webPath); ?>" loading="lazy" allowfullscreen></iframe>
|
||||||
<iframe class="col-simple" id="iRenderFrame" src="<?= UriFactory::build('{/api}orw/resource/render?id=' . $resource->id); ?>" loading="lazy" allowfullscreen></iframe>
|
<?php else : ?>
|
||||||
|
<iframe class="col-simple" id="iRenderFrame" src="<?= UriFactory::build('{/api}orw/resource/render?id=' . $resource->id); ?>" loading="lazy" sandbox="allow-forms allow-scripts" allowfullscreen></iframe>
|
||||||
|
<?php endif; ?>
|
||||||
<?php endif; ?>
|
<?php endif; ?>
|
||||||
</div>
|
</div>
|
||||||
</div>
|
</div>
|
||||||
|
|
|
||||||
Loading…
Reference in New Issue
Block a user