rector fixes + bug fixes

Admin/Install/Workflow.install.json

@@ -60,5 +60,31 @@
                 }
             }
         }
+    },
+    "workflows": {
+        "OnlineResourceWatcher": [
+            {
+                "id": "1005500005",
+                "settings": {
+                    "interval": "* 3 * * *"
+                },
+                "children": [
+                    {
+                        "id": "1008000001",
+                        "comment": "Check resources",
+                        "settings": {},
+                        "children": [
+                            {
+                                "id": "1008000002",
+                                "comment": "Inform changed ",
+                                "settings": {},
+                                "children": [
+                                ]
+                            }
+                        ]
+                    }
+                ]
+            }
+        ]
     }
 }

Controller/ApiController.php

@@ -36,6 +36,8 @@ use Modules\Admin\Models\SettingsEnum;
 use Modules\OnlineResourceWatcher\Models\SettingsEnum as OrwSettingsEnum;
 use Modules\Messages\Models\EmailMapper;
 use Modules\OnlineResourceWatcher\Models\InformBlacklistMapper;
+use phpOMS\System\OperatingSystem;
+use phpOMS\System\SystemType;
 
 /**
  * OnlineResourceWatcher controller class.
@@ -68,11 +70,9 @@ final class ApiController extends Controller
             ->execute();
 
         $path = '';
-        if (\is_dir($basePath = __DIR__ . '/' . $resource->path . '/' . $resource->lastVersionPath)) {
+        if (\is_dir($basePath = __DIR__ . '/../Files/' . $resource->path . '/' . $resource->lastVersionPath)) {
             if (\is_file($basePath . '/index.htm')) {
-                $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.htm';
+                $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.jpg';
-            } elseif (\is_file($basePath . '/index.html')) {
-                $path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.html';
             } else {
                 $files = \scandir($basePath);
                 $path  = '';
@@ -269,11 +269,15 @@ final class ApiController extends Controller
                 'loop' => 0,
             ];
 
+            try {
                 SystemUtils::runProc(
-                'wget',
+                    OperatingSystem::getSystem() === SystemType::WIN ? 'wget.exe' : 'wget',
-                '--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --page-requisites --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.html ' . $resource->uri,
+                    '--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m -U mozilla --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.htm ' . $resource->uri,
                     true
                 );
+            } catch (\Throwable $t) {
+                $this->app->logger->error($t->getMessage());
+            }
         }
 
         $handler = $this->app->moduleManager->get('Admin', 'Api')->setUpServerMailHandler();
@@ -319,10 +323,10 @@ final class ApiController extends Controller
                     $report->versionPath = (string) $check['timestamp'];
                     $report->status      = ReportStatus::DOWNLOAD_ERROR;
 
-                    ReportMapper::create()->execute($report);
+                    $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
-
+                    $old = clone $resource;
                     $resource->checkedAt = $report->createdAt;
-                    ResourceMapper::update()->execute($resource);
+                    $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
 
                     unset($toCheck[$index]);
 
@@ -332,6 +336,7 @@ final class ApiController extends Controller
                 $path = $check['path'];
                 if (!\is_dir($path)) {
                     // Either the download takes too long or the download failed!
+                    // Let's go to the next element and re-check later on.
                     continue;
                 }
 
@@ -339,15 +344,26 @@ final class ApiController extends Controller
                 $id  = (int) \substr($path, $baseLen + 1, $end - $baseLen - 1);
 
                 // new resource
-                if ($check['loop'] === $maxLoops && !\is_dir($basePath . '/' . $id)) {
                 $filesNew = \scandir($path);
                 if ($filesNew === false) {
                     $filesNew = [];
                 }
 
+                $oldPath   = '';
+                $newPath   = '';
+                $extension = '';
+
                 $fileName = '';
-                    if (\in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew)) {
+                if (\in_array('index.htm', $filesNew)
-                        $fileName = \in_array('index.htm', $filesNew) ? 'index.htm' : 'index.html';
+                    || ($hasHtml = \in_array('index.html', $filesNew))
+                ) {
+                    if ($hasHtml) {
+                        \rename($path . '/index.html', $path . '/index.htm');
+                    }
+
+                    $extension = 'htm';
+                    $fileName = 'index.htm';
+                    $toCheck[$index]['handled'] = true;
                 } else {
                     foreach ($filesNew as $file) {
                         if (StringUtils::endsWith($file, '.png')
@@ -363,11 +379,16 @@ final class ApiController extends Controller
                             || StringUtils::endsWith($file, '.txt')
                         ) {
                             $fileName = $file;
+                            $extension = \substr($file, \strripos($file, '.') + 1);
+                            $toCheck[$index]['handled'] = true;
+
                             break;
                         }
                     }
                 }
 
+                // Is new resource
+                if (!\is_dir($basePath . '/' . $id)) {
                     $report              = new Report();
                     $report->resource    = $resource->id;
                     $report->versionPath = (string) $check['timestamp'];
@@ -380,22 +401,38 @@ final class ApiController extends Controller
                         $report->status = ReportStatus::DOWNLOAD_ERROR;
                     }
 
-                    ReportMapper::create()->execute($report);
+                    $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
-
+                    $old = clone $resource;
                     $resource->path            = (string) $resource->id;
                     $resource->lastVersionPath = (string) $check['timestamp'];
                     $resource->lastVersionDate = $report->createdAt;
                     $resource->hash            = $hash == false ? '' : $hash;
                     $resource->checkedAt       = $report->createdAt;
-                    ResourceMapper::update()->execute($resource);
+                    $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
 
                     Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
                     unset($toCheck[$index]);
 
-                    continue;
+                    if ($extension === 'htm') {
+                        try {
+                            if (OperatingSystem::getSystem() === SystemType::WIN) {
+                                SystemUtils::runProc(
+                                    'firefox.exe',
+                                    '---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri,
+                                    true
+                                );
+                            } else {
+                                SystemUtils::runProc(
+                                    'xvfb-run',
+                                    '--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"',
+                                    true
+                                );
+                            }
+                        } catch (\Throwable $t) {
+                            $this->app->logger->error($t->getMessage());
+                        }
                     }
 
-                if (!\is_dir($basePath . '/' . $id)) {
                     continue;
                 }
 
@@ -417,54 +454,8 @@ final class ApiController extends Controller
                 }
 
                 $lastVersionPath = $basePath . '/' . $id . '/' . $lastVersionTimestamp;
-
+                $oldPath         = $lastVersionPath . '/' . $fileName;
-                $filesNew = \scandir($path);
+                $newPath          = $path . '/' . $fileName;
-                if ($filesNew === false) {
-                    $filesNew = [];
-                }
-
-                // Using this because the index.htm gets created last and at the time of the check below it may not yet exist.
-                $filesOld = \scandir($lastVersionPath);
-                if ($filesOld === false) {
-                    $filesOld = [];
-                }
-
-                $oldPath   = '';
-                $newPath   = '';
-                $extension = '';
-
-                if (\in_array('index.htm', $filesOld) || \in_array('index.html', $filesOld)
-                    || \in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew)
-                ) {
-                    $extension = \in_array('index.htm', $filesOld) ? 'htm' : 'html';
-                    $oldPath   = $lastVersionPath . '/index.' . $extension;
-                    $newPath   = $path . '/index.' . $extension;
-
-                    $toCheck[$index]['handled'] = true;
-                } else {
-                    foreach ($filesNew as $file) {
-                        if (StringUtils::endsWith($file, '.png')
-                            || StringUtils::endsWith($file, '.jpg')
-                            || StringUtils::endsWith($file, '.jpeg')
-                            || StringUtils::endsWith($file, '.gif')
-                            || StringUtils::endsWith($file, '.pdf')
-                            || StringUtils::endsWith($file, '.doc')
-                            || StringUtils::endsWith($file, '.docx')
-                            || StringUtils::endsWith($file, '.xls')
-                            || StringUtils::endsWith($file, '.xlsx')
-                            || StringUtils::endsWith($file, '.md')
-                            || StringUtils::endsWith($file, '.txt')
-                        ) {
-                            $oldPath   = $lastVersionPath . '/' . $file;
-                            $newPath   = $path . '/' . $file;
-                            $extension = \substr($file, \strripos($file, '.') + 1);
-
-                            $toCheck[$index]['handled'] = true;
-
-                            break;
-                        }
-                    }
-                }
 
                 if (!\is_file($newPath) || !$toCheck[$index]['handled']) {
                     continue;
@@ -491,7 +482,7 @@ final class ApiController extends Controller
 
                         // Handle xpath
                         if ($difference > 0
-                            && ($extension === 'htm' || $extension === 'html')
+                            && $extension === 'htm'
                             && $resource->path !== ''
                         ) {
                             $xmlOld = new \DOMDocument();
@@ -532,6 +523,8 @@ final class ApiController extends Controller
                 $report->versionPath  = (string) $check['timestamp'];
                 $report->changeMetric = $difference;
 
+                $old = clone $resource;
+
                 if ($difference !== 0) {
                     $report->status = ReportStatus::CHANGE;
 
@@ -544,6 +537,27 @@ final class ApiController extends Controller
 
                     Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
 
+                    // If is htm/html create image
+                    if ($extension === 'htm') {
+                        try {
+                            if (OperatingSystem::getSystem() === SystemType::WIN) {
+                                SystemUtils::runProc(
+                                    'firefox.exe',
+                                    '---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri,
+                                    true
+                                );
+                            } else {
+                                SystemUtils::runProc(
+                                    'xvfb-run',
+                                    '--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"',
+                                    true
+                                );
+                            }
+                        } catch (\Throwable $t) {
+                            $this->app->logger->error($t->getMessage());
+                        }
+                    }
+
                     // @todo: move to informUsers function
                     $owner = new Inform();
                     $owner->email = $resource->owner->getEmail();
@@ -608,10 +622,9 @@ final class ApiController extends Controller
                     }
                 }
 
-                ReportMapper::create()->execute($report);
+                $this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
-
                 $resource->checkedAt = $report->createdAt;
-                ResourceMapper::update()->execute($resource);
+                $this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
 
                 // Directory::delete($basePath . '/temp/' . $id);
 

Controller/BackendController.php

@@ -134,7 +134,9 @@ final class BackendController extends Controller
 
         /** @var \Modules\OnlineResourceWatcher\Models\Resource $resource */
         $resource = ResourceMapper::get()
+            ->with('reports')
             ->where('id', (int) $request->getData('id'))
+            ->limit(25, 'reports')
             ->execute();
 
         $view->setData('resource', $resource);

Models/Resource.php

@@ -107,6 +107,8 @@ class Resource implements \JsonSerializable
      */
     public array $inform = [];
 
+    public array $reports = [];
+
     /**
      * Organization.
      *

Models/ResourceMapper.php

@@ -82,6 +82,12 @@ final class ResourceMapper extends DataMapperFactory
             'self'     => 'orw_resource_info_resource',
             'external' => null,
         ],
+        'reports' => [
+            'mapper'   => ReportMapper::class,
+            'table'    => 'orw_resource_report',
+            'self'     => 'orw_resource_report_resource',
+            'external' => null,
+        ],
     ];
 
     /**

Theme/Backend/Lang/en.lang.php

@@ -87,4 +87,8 @@ return ['OnlineResourceWatcher' => [
     'XPath'          => 'XPath',
     'Comparison'          => 'Comparison',
     'History'          => 'History',
+    'rstatus-1'          => 'Resource added',
+    'rstatus-2'          => 'No change',
+    'rstatus-3'          => 'Changed',
+    'rstatus--1'          => 'Download error',
 ]];

Theme/Backend/resource-list.tpl.php

@@ -44,7 +44,7 @@ $next = $tableView->getNextLink(
                 <?= $tableView->renderTitle(
                     $this->getHtml('Resources')
                 ); ?>
-				<a class="button rf save" href="<?= UriFactory::build('{/base}/'); ?>orw/resources/create"><?= $this->getHtml('New', '0', '0'); ?></a>
+                <a class="button button end-xs save" href="<?= UriFactory::build('{/base}/'); ?>orw/resources/create"><?= $this->getHtml('New', '0', '0'); ?></a>
             </div>
             <div class="slider">
             <table id="<?= $tableView->id; ?>" class="default sticky">

Theme/Backend/resource-single.tpl.php

@@ -16,6 +16,7 @@ use phpOMS\Uri\UriFactory;
 
 /** @var \Modules\OnlineResourceWatcher\Models\Resource */
 $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Models\NullResource();
+$reports = $resource->reports;
 ?>
 <div class="row">
     <div class="col-xs-8">
@@ -56,8 +57,18 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
     <div class="col-xs-4">
         <div class="portlet">
             <div class="portlet-head"><?= $this->getHtml('History'); ?></div>
-			<div class="portlet-body">
+            <table class="default">
-			</div>
+				<thead>
+					<tr>
+						<td><?= $this->getHtml('Date'); ?>
+						<td><?= $this->getHtml('Status'); ?>
+				<tbody>
+					<?php foreach ($reports as $report) : ?>
+					<tr>
+						<td><?= $this->printHtml($report->createdAt->format('Y-m-d')); ?>
+						<td><?= $this->getHtml('rstatus-' . $report->status); ?>
+					<?php endforeach; ?>
+			</table>
         </div>
     </div>
 </div>