diff --git a/Admin/Install/Workflow.install.json b/Admin/Install/Workflow.install.json index 38eef25..ec06d47 100755 --- a/Admin/Install/Workflow.install.json +++ b/Admin/Install/Workflow.install.json @@ -60,5 +60,31 @@ } } } + }, + "workflows": { + "OnlineResourceWatcher": [ + { + "id": "1005500005", + "settings": { + "interval": "* 3 * * *" + }, + "children": [ + { + "id": "1008000001", + "comment": "Check resources", + "settings": {}, + "children": [ + { + "id": "1008000002", + "comment": "Inform changed ", + "settings": {}, + "children": [ + ] + } + ] + } + ] + } + ] } } \ No newline at end of file diff --git a/Controller/ApiController.php b/Controller/ApiController.php index 04fe648..2877427 100755 --- a/Controller/ApiController.php +++ b/Controller/ApiController.php @@ -36,6 +36,8 @@ use Modules\Admin\Models\SettingsEnum; use Modules\OnlineResourceWatcher\Models\SettingsEnum as OrwSettingsEnum; use Modules\Messages\Models\EmailMapper; use Modules\OnlineResourceWatcher\Models\InformBlacklistMapper; +use phpOMS\System\OperatingSystem; +use phpOMS\System\SystemType; /** * OnlineResourceWatcher controller class. @@ -68,11 +70,9 @@ final class ApiController extends Controller ->execute(); $path = ''; - if (\is_dir($basePath = __DIR__ . '/' . $resource->path . '/' . $resource->lastVersionPath)) { + if (\is_dir($basePath = __DIR__ . '/../Files/' . $resource->path . '/' . $resource->lastVersionPath)) { if (\is_file($basePath . '/index.htm')) { - $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.htm'; - } elseif (\is_file($basePath . '/index.html')) { - $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.html'; + $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.jpg'; } else { $files = \scandir($basePath); $path = ''; @@ -269,11 +269,15 @@ final class ApiController extends Controller 'loop' => 0, ]; - SystemUtils::runProc( - 'wget', - '--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --page-requisites --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.html ' . $resource->uri, - true - ); + try { + SystemUtils::runProc( + OperatingSystem::getSystem() === SystemType::WIN ? 'wget.exe' : 'wget', + '--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m -U mozilla --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.htm ' . $resource->uri, + true + ); + } catch (\Throwable $t) { + $this->app->logger->error($t->getMessage()); + } } $handler = $this->app->moduleManager->get('Admin', 'Api')->setUpServerMailHandler(); @@ -319,10 +323,10 @@ final class ApiController extends Controller $report->versionPath = (string) $check['timestamp']; $report->status = ReportStatus::DOWNLOAD_ERROR; - ReportMapper::create()->execute($report); - + $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin()); + $old = clone $resource; $resource->checkedAt = $report->createdAt; - ResourceMapper::update()->execute($resource); + $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin()); unset($toCheck[$index]); @@ -332,6 +336,7 @@ final class ApiController extends Controller $path = $check['path']; if (!\is_dir($path)) { // Either the download takes too long or the download failed! + // Let's go to the next element and re-check later on. continue; } @@ -339,35 +344,51 @@ final class ApiController extends Controller $id = (int) \substr($path, $baseLen + 1, $end - $baseLen - 1); // new resource - if ($check['loop'] === $maxLoops && !\is_dir($basePath . '/' . $id)) { - $filesNew = \scandir($path); - if ($filesNew === false) { - $filesNew = []; + $filesNew = \scandir($path); + if ($filesNew === false) { + $filesNew = []; + } + + $oldPath = ''; + $newPath = ''; + $extension = ''; + + $fileName = ''; + if (\in_array('index.htm', $filesNew) + || ($hasHtml = \in_array('index.html', $filesNew)) + ) { + if ($hasHtml) { + \rename($path . '/index.html', $path . '/index.htm'); } - $fileName = ''; - if (\in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew)) { - $fileName = \in_array('index.htm', $filesNew) ? 'index.htm' : 'index.html'; - } else { - foreach ($filesNew as $file) { - if (StringUtils::endsWith($file, '.png') - || StringUtils::endsWith($file, '.jpg') - || StringUtils::endsWith($file, '.jpeg') - || StringUtils::endsWith($file, '.gif') - || StringUtils::endsWith($file, '.pdf') - || StringUtils::endsWith($file, '.doc') - || StringUtils::endsWith($file, '.docx') - || StringUtils::endsWith($file, '.xls') - || StringUtils::endsWith($file, '.xlsx') - || StringUtils::endsWith($file, '.md') - || StringUtils::endsWith($file, '.txt') - ) { - $fileName = $file; - break; - } + $extension = 'htm'; + $fileName = 'index.htm'; + $toCheck[$index]['handled'] = true; + } else { + foreach ($filesNew as $file) { + if (StringUtils::endsWith($file, '.png') + || StringUtils::endsWith($file, '.jpg') + || StringUtils::endsWith($file, '.jpeg') + || StringUtils::endsWith($file, '.gif') + || StringUtils::endsWith($file, '.pdf') + || StringUtils::endsWith($file, '.doc') + || StringUtils::endsWith($file, '.docx') + || StringUtils::endsWith($file, '.xls') + || StringUtils::endsWith($file, '.xlsx') + || StringUtils::endsWith($file, '.md') + || StringUtils::endsWith($file, '.txt') + ) { + $fileName = $file; + $extension = \substr($file, \strripos($file, '.') + 1); + $toCheck[$index]['handled'] = true; + + break; } } + } + // Is new resource + if (!\is_dir($basePath . '/' . $id)) { $report = new Report(); $report->resource = $resource->id; $report->versionPath = (string) $check['timestamp']; @@ -380,22 +401,38 @@ final class ApiController extends Controller $report->status = ReportStatus::DOWNLOAD_ERROR; } - ReportMapper::create()->execute($report); - + $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin()); + $old = clone $resource; $resource->path = (string) $resource->id; $resource->lastVersionPath = (string) $check['timestamp']; $resource->lastVersionDate = $report->createdAt; $resource->hash = $hash == false ? '' : $hash; $resource->checkedAt = $report->createdAt; - ResourceMapper::update()->execute($resource); + $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin()); Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']); unset($toCheck[$index]); - continue; - } + if ($extension === 'htm') { + try { + if (OperatingSystem::getSystem() === SystemType::WIN) { + SystemUtils::runProc( + 'firefox.exe', + '---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri, + true + ); + } else { + SystemUtils::runProc( + 'xvfb-run', + '--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"', + true + ); + } + } catch (\Throwable $t) { + $this->app->logger->error($t->getMessage()); + } + } - if (!\is_dir($basePath . '/' . $id)) { continue; } @@ -417,54 +454,8 @@ final class ApiController extends Controller } $lastVersionPath = $basePath . '/' . $id . '/' . $lastVersionTimestamp; - - $filesNew = \scandir($path); - if ($filesNew === false) { - $filesNew = []; - } - - // Using this because the index.htm gets created last and at the time of the check below it may not yet exist. - $filesOld = \scandir($lastVersionPath); - if ($filesOld === false) { - $filesOld = []; - } - - $oldPath = ''; - $newPath = ''; - $extension = ''; - - if (\in_array('index.htm', $filesOld) || \in_array('index.html', $filesOld) - || \in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew) - ) { - $extension = \in_array('index.htm', $filesOld) ? 'htm' : 'html'; - $oldPath = $lastVersionPath . '/index.' . $extension; - $newPath = $path . '/index.' . $extension; - - $toCheck[$index]['handled'] = true; - } else { - foreach ($filesNew as $file) { - if (StringUtils::endsWith($file, '.png') - || StringUtils::endsWith($file, '.jpg') - || StringUtils::endsWith($file, '.jpeg') - || StringUtils::endsWith($file, '.gif') - || StringUtils::endsWith($file, '.pdf') - || StringUtils::endsWith($file, '.doc') - || StringUtils::endsWith($file, '.docx') - || StringUtils::endsWith($file, '.xls') - || StringUtils::endsWith($file, '.xlsx') - || StringUtils::endsWith($file, '.md') - || StringUtils::endsWith($file, '.txt') - ) { - $oldPath = $lastVersionPath . '/' . $file; - $newPath = $path . '/' . $file; - $extension = \substr($file, \strripos($file, '.') + 1); - - $toCheck[$index]['handled'] = true; - - break; - } - } - } + $oldPath = $lastVersionPath . '/' . $fileName; + $newPath = $path . '/' . $fileName; if (!\is_file($newPath) || !$toCheck[$index]['handled']) { continue; @@ -491,7 +482,7 @@ final class ApiController extends Controller // Handle xpath if ($difference > 0 - && ($extension === 'htm' || $extension === 'html') + && $extension === 'htm' && $resource->path !== '' ) { $xmlOld = new \DOMDocument(); @@ -532,6 +523,8 @@ final class ApiController extends Controller $report->versionPath = (string) $check['timestamp']; $report->changeMetric = $difference; + $old = clone $resource; + if ($difference !== 0) { $report->status = ReportStatus::CHANGE; @@ -544,6 +537,27 @@ final class ApiController extends Controller Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']); + // If is htm/html create image + if ($extension === 'htm') { + try { + if (OperatingSystem::getSystem() === SystemType::WIN) { + SystemUtils::runProc( + 'firefox.exe', + '---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri, + true + ); + } else { + SystemUtils::runProc( + 'xvfb-run', + '--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"', + true + ); + } + } catch (\Throwable $t) { + $this->app->logger->error($t->getMessage()); + } + } + // @todo: move to informUsers function $owner = new Inform(); $owner->email = $resource->owner->getEmail(); @@ -608,10 +622,9 @@ final class ApiController extends Controller } } - ReportMapper::create()->execute($report); - + $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin()); $resource->checkedAt = $report->createdAt; - ResourceMapper::update()->execute($resource); + $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin()); // Directory::delete($basePath . '/temp/' . $id); diff --git a/Controller/BackendController.php b/Controller/BackendController.php index 4cc14c6..7662505 100755 --- a/Controller/BackendController.php +++ b/Controller/BackendController.php @@ -134,7 +134,9 @@ final class BackendController extends Controller /** @var \Modules\OnlineResourceWatcher\Models\Resource $resource */ $resource = ResourceMapper::get() + ->with('reports') ->where('id', (int) $request->getData('id')) + ->limit(25, 'reports') ->execute(); $view->setData('resource', $resource); diff --git a/Models/Resource.php b/Models/Resource.php index 3c316a6..909c6f8 100755 --- a/Models/Resource.php +++ b/Models/Resource.php @@ -107,6 +107,8 @@ class Resource implements \JsonSerializable */ public array $inform = []; + public array $reports = []; + /** * Organization. * diff --git a/Models/ResourceMapper.php b/Models/ResourceMapper.php index c951d90..b26b5bc 100755 --- a/Models/ResourceMapper.php +++ b/Models/ResourceMapper.php @@ -82,6 +82,12 @@ final class ResourceMapper extends DataMapperFactory 'self' => 'orw_resource_info_resource', 'external' => null, ], + 'reports' => [ + 'mapper' => ReportMapper::class, + 'table' => 'orw_resource_report', + 'self' => 'orw_resource_report_resource', + 'external' => null, + ], ]; /** diff --git a/Theme/Backend/Lang/en.lang.php b/Theme/Backend/Lang/en.lang.php index 14ee961..68853c9 100755 --- a/Theme/Backend/Lang/en.lang.php +++ b/Theme/Backend/Lang/en.lang.php @@ -87,4 +87,8 @@ return ['OnlineResourceWatcher' => [ 'XPath' => 'XPath', 'Comparison' => 'Comparison', 'History' => 'History', + 'rstatus-1' => 'Resource added', + 'rstatus-2' => 'No change', + 'rstatus-3' => 'Changed', + 'rstatus--1' => 'Download error', ]]; diff --git a/Theme/Backend/resource-create.tpl.php b/Theme/Backend/resource-create.tpl.php index 1a0672e..dd139df 100755 --- a/Theme/Backend/resource-create.tpl.php +++ b/Theme/Backend/resource-create.tpl.php @@ -15,34 +15,34 @@ declare(strict_types=1); use phpOMS\Uri\UriFactory; ?>
-
-
-
-
getHtml('CreateResource', '0', '0'); ?>
-
-
- - -
+
+
+ +
getHtml('CreateResource', '0', '0'); ?>
+
+
+ + +
-
- - -
+
+ + +
-
- - -
-
-
- -
- -
-
+
+ + +
+
+
+ +
+ +
+
-
+
getHtml('Inform', '0', '0'); ?>
diff --git a/Theme/Backend/resource-list.tpl.php b/Theme/Backend/resource-list.tpl.php index 1b4afd2..f3f2614 100755 --- a/Theme/Backend/resource-list.tpl.php +++ b/Theme/Backend/resource-list.tpl.php @@ -43,8 +43,8 @@ $next = $tableView->getNextLink(
renderTitle( $this->getHtml('Resources') - ); ?> - getHtml('New', '0', '0'); ?> + ); ?> + getHtml('New', '0', '0'); ?>
@@ -54,18 +54,18 @@ $next = $tableView->getNextLink( 'id', $this->getHtml('ID', '0', '0'), 'number' - ); ?> -
renderHeaderElement( + ); ?> + renderHeaderElement( 'resource', $this->getHtml('Resource'), 'text' ); ?> - renderHeaderElement( + renderHeaderElement( 'status', $this->getHtml('Status'), 'text' - ); ?> - renderHeaderElement( + ); ?> + renderHeaderElement( 'lastChecked', $this->getHtml('Checked'), 'date' diff --git a/Theme/Backend/resource-single.tpl.php b/Theme/Backend/resource-single.tpl.php index 3853d59..d247d89 100755 --- a/Theme/Backend/resource-single.tpl.php +++ b/Theme/Backend/resource-single.tpl.php @@ -16,57 +16,68 @@ use phpOMS\Uri\UriFactory; /** @var \Modules\OnlineResourceWatcher\Models\Resource */ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Models\NullResource(); +$reports = $resource->reports; ?>
-
-
- -
getHtml('Resource'); ?>
-
-
- - -
+
+
+ +
getHtml('Resource'); ?>
+
+
+ + +
-
- - -
+
-
- - -
+
+ + +
-
- - -
-
-
- -
- -
-
+
+ + +
+
+
+ +
+ +
+
-
-
-
getHtml('History'); ?>
-
-
-
-
+
+
+
getHtml('History'); ?>
+ + + + + + +
getHtml('Date'); ?> + getHtml('Status'); ?> +
printHtml($report->createdAt->format('Y-m-d')); ?> + getHtml('rstatus-' . $report->status); ?> + +
+
+
-
-
-
-
+
+
+
+
  • @@ -74,15 +85,15 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
    -
    -
    -
    -
    - -
    -
    -
    -
    +
    +
    +
    +
    + +
    +
    +
    +
    @@ -91,7 +102,7 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
-
-
-
+
+
+