Jingga/oms-OnlineResourceWatcher
1
0
Fork 0
mirror of https://github.com/Karaka-Management/oms-OnlineResourceWatcher.git synced 2026-01-31 08:48:41 +00:00
Code Issues Actions Packages Projects Releases Wiki Activity

rector fixes + bug fixes

Browse Source
This commit is contained in:
Dennis Eichhorn 2023-05-27 03:06:46 +00:00
parent 6f79d04171
commit 738fbf489d
9 changed files with 241 additions and 177 deletions
Show Stats Download Patch File Download Diff File Expand all files Collapse all files

26
Admin/Install/Workflow.install.json
View File

@ -60,5 +60,31 @@
}
}
}
},
"workflows": {
"OnlineResourceWatcher": [
{
"id": "1005500005",
"settings": {
"interval": "* 3 * * *"
},
"children": [
{
"id": "1008000001",
"comment": "Check resources",
"settings": {},
"children": [
{
"id": "1008000002",
"comment": "Inform changed ",
"settings": {},
"children": [
]
}
]
}
]
}
]
}
}

201
Controller/ApiController.php
View File

@ -36,6 +36,8 @@ use Modules\Admin\Models\SettingsEnum;
use Modules\OnlineResourceWatcher\Models\SettingsEnum as OrwSettingsEnum;
use Modules\Messages\Models\EmailMapper;
use Modules\OnlineResourceWatcher\Models\InformBlacklistMapper;
use phpOMS\System\OperatingSystem;
use phpOMS\System\SystemType;
/**
* OnlineResourceWatcher controller class.
@ -68,11 +70,9 @@ final class ApiController extends Controller
->execute();
$path = '';
if (\is_dir($basePath = __DIR__ . '/' . $resource->path . '/' . $resource->lastVersionPath)) {
if (\is_dir($basePath = __DIR__ . '/../Files/' . $resource->path . '/' . $resource->lastVersionPath)) {
if (\is_file($basePath . '/index.htm')) {
$path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.htm';
} elseif (\is_file($basePath . '/index.html')) {
$path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.html';
$path = 'Modules/OnlineResourceWatcher/Files/' . $resource->path . '/' . $resource->lastVersionPath . '/index.jpg';
} else {
$files = \scandir($basePath);
$path = '';
@ -269,11 +269,15 @@ final class ApiController extends Controller
'loop' => 0,
];
SystemUtils::runProc(
'wget',
'--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --page-requisites --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.html ' . $resource->uri,
true
);
try {
SystemUtils::runProc(
OperatingSystem::getSystem() === SystemType::WIN ? 'wget.exe' : 'wget',
'--retry-connrefused --waitretry=1 --read-timeout=10 --timeout=10 --dns-timeout=10 -t 2 --quota=25m --adjust-extension --span-hosts --convert-links --no-directories --restrict-file-names=windows --no-parent ‐‐execute robots=off --limit-rate=5m -U mozilla --accept css,png,jpg,jpeg,gif,htm,html,txt,md,pdf,xls,xlsx,doc,docx --directory-prefix=' . $path . ' ‐‐output-document=index.htm ' . $resource->uri,
true
);
} catch (\Throwable $t) {
$this->app->logger->error($t->getMessage());
}
}
$handler = $this->app->moduleManager->get('Admin', 'Api')->setUpServerMailHandler();
@ -319,10 +323,10 @@ final class ApiController extends Controller
$report->versionPath = (string) $check['timestamp'];
$report->status = ReportStatus::DOWNLOAD_ERROR;
ReportMapper::create()->execute($report);
$this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
$old = clone $resource;
$resource->checkedAt = $report->createdAt;
ResourceMapper::update()->execute($resource);
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
unset($toCheck[$index]);
@ -332,6 +336,7 @@ final class ApiController extends Controller
$path = $check['path'];
if (!\is_dir($path)) {
// Either the download takes too long or the download failed!
// Let's go to the next element and re-check later on.
continue;
}
@ -339,35 +344,51 @@ final class ApiController extends Controller
$id = (int) \substr($path, $baseLen + 1, $end - $baseLen - 1);
// new resource
if ($check['loop'] === $maxLoops && !\is_dir($basePath . '/' . $id)) {
$filesNew = \scandir($path);
if ($filesNew === false) {
$filesNew = [];
$filesNew = \scandir($path);
if ($filesNew === false) {
$filesNew = [];
}
$oldPath = '';
$newPath = '';
$extension = '';
$fileName = '';
if (\in_array('index.htm', $filesNew)
|| ($hasHtml = \in_array('index.html', $filesNew))
) {
if ($hasHtml) {
\rename($path . '/index.html', $path . '/index.htm');
}
$fileName = '';
if (\in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew)) {
$fileName = \in_array('index.htm', $filesNew) ? 'index.htm' : 'index.html';
} else {
foreach ($filesNew as $file) {
if (StringUtils::endsWith($file, '.png')
|| StringUtils::endsWith($file, '.jpg')
|| StringUtils::endsWith($file, '.jpeg')
|| StringUtils::endsWith($file, '.gif')
|| StringUtils::endsWith($file, '.pdf')
|| StringUtils::endsWith($file, '.doc')
|| StringUtils::endsWith($file, '.docx')
|| StringUtils::endsWith($file, '.xls')
|| StringUtils::endsWith($file, '.xlsx')
|| StringUtils::endsWith($file, '.md')
|| StringUtils::endsWith($file, '.txt')
) {
$fileName = $file;
break;
}
$extension = 'htm';
$fileName = 'index.htm';
$toCheck[$index]['handled'] = true;
} else {
foreach ($filesNew as $file) {
if (StringUtils::endsWith($file, '.png')
|| StringUtils::endsWith($file, '.jpg')
|| StringUtils::endsWith($file, '.jpeg')
|| StringUtils::endsWith($file, '.gif')
|| StringUtils::endsWith($file, '.pdf')
|| StringUtils::endsWith($file, '.doc')
|| StringUtils::endsWith($file, '.docx')
|| StringUtils::endsWith($file, '.xls')
|| StringUtils::endsWith($file, '.xlsx')
|| StringUtils::endsWith($file, '.md')
|| StringUtils::endsWith($file, '.txt')
) {
$fileName = $file;
$extension = \substr($file, \strripos($file, '.') + 1);
$toCheck[$index]['handled'] = true;
break;
}
}
}
// Is new resource
if (!\is_dir($basePath . '/' . $id)) {
$report = new Report();
$report->resource = $resource->id;
$report->versionPath = (string) $check['timestamp'];
@ -380,22 +401,38 @@ final class ApiController extends Controller
$report->status = ReportStatus::DOWNLOAD_ERROR;
}
ReportMapper::create()->execute($report);
$this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
$old = clone $resource;
$resource->path = (string) $resource->id;
$resource->lastVersionPath = (string) $check['timestamp'];
$resource->lastVersionDate = $report->createdAt;
$resource->hash = $hash == false ? '' : $hash;
$resource->checkedAt = $report->createdAt;
ResourceMapper::update()->execute($resource);
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
unset($toCheck[$index]);
continue;
}
if ($extension === 'htm') {
try {
if (OperatingSystem::getSystem() === SystemType::WIN) {
SystemUtils::runProc(
'firefox.exe',
'---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri,
true
);
} else {
SystemUtils::runProc(
'xvfb-run',
'--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"',
true
);
}
} catch (\Throwable $t) {
$this->app->logger->error($t->getMessage());
}
}
if (!\is_dir($basePath . '/' . $id)) {
continue;
}
@ -417,54 +454,8 @@ final class ApiController extends Controller
}
$lastVersionPath = $basePath . '/' . $id . '/' . $lastVersionTimestamp;
$filesNew = \scandir($path);
if ($filesNew === false) {
$filesNew = [];
}
// Using this because the index.htm gets created last and at the time of the check below it may not yet exist.
$filesOld = \scandir($lastVersionPath);
if ($filesOld === false) {
$filesOld = [];
}
$oldPath = '';
$newPath = '';
$extension = '';
if (\in_array('index.htm', $filesOld) || \in_array('index.html', $filesOld)
|| \in_array('index.htm', $filesNew) || \in_array('index.html', $filesNew)
) {
$extension = \in_array('index.htm', $filesOld) ? 'htm' : 'html';
$oldPath = $lastVersionPath . '/index.' . $extension;
$newPath = $path . '/index.' . $extension;
$toCheck[$index]['handled'] = true;
} else {
foreach ($filesNew as $file) {
if (StringUtils::endsWith($file, '.png')
|| StringUtils::endsWith($file, '.jpg')
|| StringUtils::endsWith($file, '.jpeg')
|| StringUtils::endsWith($file, '.gif')
|| StringUtils::endsWith($file, '.pdf')
|| StringUtils::endsWith($file, '.doc')
|| StringUtils::endsWith($file, '.docx')
|| StringUtils::endsWith($file, '.xls')
|| StringUtils::endsWith($file, '.xlsx')
|| StringUtils::endsWith($file, '.md')
|| StringUtils::endsWith($file, '.txt')
) {
$oldPath = $lastVersionPath . '/' . $file;
$newPath = $path . '/' . $file;
$extension = \substr($file, \strripos($file, '.') + 1);
$toCheck[$index]['handled'] = true;
break;
}
}
}
$oldPath = $lastVersionPath . '/' . $fileName;
$newPath = $path . '/' . $fileName;
if (!\is_file($newPath) || !$toCheck[$index]['handled']) {
continue;
@ -491,7 +482,7 @@ final class ApiController extends Controller
// Handle xpath
if ($difference > 0
&& ($extension === 'htm' || $extension === 'html')
&& $extension === 'htm'
&& $resource->path !== ''
) {
$xmlOld = new \DOMDocument();
@ -532,6 +523,8 @@ final class ApiController extends Controller
$report->versionPath = (string) $check['timestamp'];
$report->changeMetric = $difference;
$old = clone $resource;
if ($difference !== 0) {
$report->status = ReportStatus::CHANGE;
@ -544,6 +537,27 @@ final class ApiController extends Controller
Directory::copy($path, $basePath . '/' . $id . '/' . $check['timestamp']);
// If is htm/html create image
if ($extension === 'htm') {
try {
if (OperatingSystem::getSystem() === SystemType::WIN) {
SystemUtils::runProc(
'firefox.exe',
'---headless --screenshot "' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg" ' . $resource->uri,
true
);
} else {
SystemUtils::runProc(
'xvfb-run',
'--server-args="-screen 0, 1920x1080x24" cutycapt --min-width=1024 --url="' . $resource->uri . '" --out="' . $basePath . '/' . $id . '/' . $check['timestamp'] . '/index.jpg"',
true
);
}
} catch (\Throwable $t) {
$this->app->logger->error($t->getMessage());
}
}
// @todo: move to informUsers function
$owner = new Inform();
$owner->email = $resource->owner->getEmail();
@ -608,10 +622,9 @@ final class ApiController extends Controller
}
}
ReportMapper::create()->execute($report);
$this->createModel($request->header->account, $report, ReportMapper::class, 'report', $request->getOrigin());
$resource->checkedAt = $report->createdAt;
ResourceMapper::update()->execute($resource);
$this->updateModel($request->header->account, $old, $resource, ResourceMapper::class, 'resource', $request->getOrigin());
// Directory::delete($basePath . '/temp/' . $id);

2
Controller/BackendController.php
View File

@ -134,7 +134,9 @@ final class BackendController extends Controller
/** @var \Modules\OnlineResourceWatcher\Models\Resource $resource */
$resource = ResourceMapper::get()
->with('reports')
->where('id', (int) $request->getData('id'))
->limit(25, 'reports')
->execute();
$view->setData('resource', $resource);

2
Models/Resource.php
View File

@ -107,6 +107,8 @@ class Resource implements \JsonSerializable
*/
public array $inform = [];
public array $reports = [];
/**
* Organization.
*

6
Models/ResourceMapper.php
View File

@ -82,6 +82,12 @@ final class ResourceMapper extends DataMapperFactory
'self' => 'orw_resource_info_resource',
'external' => null,
],
'reports' => [
'mapper' => ReportMapper::class,
'table' => 'orw_resource_report',
'self' => 'orw_resource_report_resource',
'external' => null,
],
];
/**

4
Theme/Backend/Lang/en.lang.php
View File

@ -87,4 +87,8 @@ return ['OnlineResourceWatcher' => [
'XPath' => 'XPath',
'Comparison' => 'Comparison',
'History' => 'History',
'rstatus-1' => 'Resource added',
'rstatus-2' => 'No change',
'rstatus-3' => 'Changed',
'rstatus--1' => 'Download error',
]];

50
Theme/Backend/resource-create.tpl.php
View File

@ -15,34 +15,34 @@ declare(strict_types=1);
use phpOMS\Uri\UriFactory;
?>
<div class="row">
<div class="col-xs-12 col-md-6">
<div class="portlet">
<form id="iUserSettings" action="<?= UriFactory::build('{/api}user/settings'); ?>" method="post">
<div class="portlet-head"><?= $this->getHtml('CreateResource', '0', '0'); ?></div>
<div class="portlet-body">
<div class="form-group">
<label for="iLogin"><?= $this->getHtml('Url', '0', '0'); ?></label>
<input id="iLogin" name="rul" type="text" required>
</div>
<div class="col-xs-12 col-md-6">
<div class="portlet">
<form id="iUserSettings" action="<?= UriFactory::build('{/api}user/settings'); ?>" method="post">
<div class="portlet-head"><?= $this->getHtml('CreateResource', '0', '0'); ?></div>
<div class="portlet-body">
<div class="form-group">
<label for="iLogin"><?= $this->getHtml('Url', '0', '0'); ?></label>
<input id="iLogin" name="rul" type="text" required>
</div>
<div class="form-group">
<label for="iElement"><?= $this->getHtml('Element', '0', '0'); ?></label>
<input id="iElement" name="element" type="text">
</div>
<div class="form-group">
<label for="iElement"><?= $this->getHtml('Element', '0', '0'); ?></label>
<input id="iElement" name="element" type="text">
</div>
<div class="form-group">
<label for="iHeader"><?= $this->getHtml('Header', '0', '0'); ?></label>
<textarea id="iHeader" name="header"></textarea>
</div>
</div>
<div class="portlet-foot">
<input id="iSubmitUser" name="submitUser" type="submit" value="<?= $this->getHtml('Create', '0', '0'); ?>">
</div>
</form>
</div>
</div>
<div class="form-group">
<label for="iHeader"><?= $this->getHtml('Header', '0', '0'); ?></label>
<textarea id="iHeader" name="header"></textarea>
</div>
</div>
<div class="portlet-foot">
<input id="iSubmitUser" name="submitUser" type="submit" value="<?= $this->getHtml('Create', '0', '0'); ?>">
</div>
</form>
</div>
</div>
<div class="col-xs-12 col-md-6">
<div class="col-xs-12 col-md-6">
<div class="portlet">
<form id="iAddAccountToGroup" action="<?= UriFactory::build('{/api}admin/group/account'); ?>" method="put">
<div class="portlet-head"><?= $this->getHtml('Inform', '0', '0'); ?></div>

14
Theme/Backend/resource-list.tpl.php
View File

@ -43,8 +43,8 @@ $next = $tableView->getNextLink(
<div class="portlet-head">
<?= $tableView->renderTitle(
$this->getHtml('Resources')
); ?>
<a class="button rf save" href="<?= UriFactory::build('{/base}/'); ?>orw/resources/create"><?= $this->getHtml('New', '0', '0'); ?></a>
); ?>
<a class="button button end-xs save" href="<?= UriFactory::build('{/base}/'); ?>orw/resources/create"><?= $this->getHtml('New', '0', '0'); ?></a>
</div>
<div class="slider">
<table id="<?= $tableView->id; ?>" class="default sticky">
@ -54,18 +54,18 @@ $next = $tableView->getNextLink(
'id',
$this->getHtml('ID', '0', '0'),
'number'
); ?>
<td class="wf-100"><?= $tableView->renderHeaderElement(
); ?>
<td class="wf-100"><?= $tableView->renderHeaderElement(
'resource',
$this->getHtml('Resource'),
'text'
); ?>
<td><?= $tableView->renderHeaderElement(
<td><?= $tableView->renderHeaderElement(
'status',
$this->getHtml('Status'),
'text'
); ?>
<td><?= $tableView->renderHeaderElement(
); ?>
<td><?= $tableView->renderHeaderElement(
'lastChecked',
$this->getHtml('Checked'),
'date'

113
Theme/Backend/resource-single.tpl.php
View File

@ -16,57 +16,68 @@ use phpOMS\Uri\UriFactory;
/** @var \Modules\OnlineResourceWatcher\Models\Resource */
$resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Models\NullResource();
$reports = $resource->reports;
?>
<div class="row">
<div class="col-xs-8">
<div class="portlet">
<form id="iResource" action="<?= UriFactory::build('{/api}resource'); ?>" method="post">
<div class="portlet-head"><?= $this->getHtml('Resource'); ?></div>
<div class="portlet-body">
<div class="form-group">
<label for="iName"><?= $this->getHtml('Name'); ?></label>
<input id="iName" name="name" type="text" value="<?= $this->printHtml($resource->title); ?>">
</div>
<div class="col-xs-8">
<div class="portlet">
<form id="iResource" action="<?= UriFactory::build('{/api}resource'); ?>" method="post">
<div class="portlet-head"><?= $this->getHtml('Resource'); ?></div>
<div class="portlet-body">
<div class="form-group">
<label for="iName"><?= $this->getHtml('Name'); ?></label>
<input id="iName" name="name" type="text" value="<?= $this->printHtml($resource->title); ?>">
</div>
<div class="form-group">
<label for="iStatus"><?= $this->getHtml('Status'); ?></label>
<select id="iStatus" name="status">
<div class="form-group">
<label for="iStatus"><?= $this->getHtml('Status'); ?></label>
<select id="iStatus" name="status">
<option value="1"<?= $resource->status === 1 ? ' selected' : ''; ?>>Active</option>
<option value="2"<?= $resource->status === 2 ? ' selected' : ''; ?>>Inactive</option>
</select>
</div>
</div>
<div class="form-group">
<label for="iUrl"><?= $this->getHtml('Url'); ?></label>
<input id="iUrl" name="uri" type="text" value="<?= $this->printHtml($resource->uri); ?>" required>
</div>
<div class="form-group">
<label for="iUrl"><?= $this->getHtml('Url'); ?></label>
<input id="iUrl" name="uri" type="text" value="<?= $this->printHtml($resource->uri); ?>" required>
</div>
<div class="form-group">
<label for="iXPath"><?= $this->getHtml('XPath'); ?></label>
<input id="iXPath" name="xpath" type="text" value="<?= $this->printHtml($resource->xpath); ?>">
</div>
</div>
<div class="portlet-foot">
<input id="iSubmitUser" name="submitUser" type="submit" value="<?= $this->getHtml('Save', '0', '0'); ?>">
</div>
</form>
</div>
</div>
<div class="form-group">
<label for="iXPath"><?= $this->getHtml('XPath'); ?></label>
<input id="iXPath" name="xpath" type="text" value="<?= $this->printHtml($resource->xpath); ?>">
</div>
</div>
<div class="portlet-foot">
<input id="iSubmitUser" name="submitUser" type="submit" value="<?= $this->getHtml('Save', '0', '0'); ?>">
</div>
</form>
</div>
</div>
<div class="col-xs-4">
<div class="portlet">
<div class="portlet-head"><?= $this->getHtml('History'); ?></div>
<div class="portlet-body">
</div>
</div>
</div>
<div class="col-xs-4">
<div class="portlet">
<div class="portlet-head"><?= $this->getHtml('History'); ?></div>
<table class="default">
<thead>
<tr>
<td><?= $this->getHtml('Date'); ?>
<td><?= $this->getHtml('Status'); ?>
<tbody>
<?php foreach ($reports as $report) : ?>
<tr>
<td><?= $this->printHtml($report->createdAt->format('Y-m-d')); ?>
<td><?= $this->getHtml('rstatus-' . $report->status); ?>
<?php endforeach; ?>
</table>
</div>
</div>
</div>
<div class="row col-simple">
<div class="col-xs-12 col-simple">
<div class="portlet col-simple">
<div class="portlet-body col-simple">
<div id="resource" class="tabview tab-2 m-editor col-simple">
<div class="col-xs-12 col-simple">
<div class="portlet col-simple">
<div class="portlet-body col-simple">
<div id="resource" class="tabview tab-2 m-editor col-simple">
<ul class="tab-links">
<li><label tabindex="0" for="resource-c-tab-1"><?= $this->getHtml('Preview'); ?></label>
<li><label tabindex="1" for="resource-c-tab-2"><?= $this->getHtml('Comparison'); ?></label>
@ -74,15 +85,15 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
<div class="tab-content col-simple">
<input type="radio" id="resource-c-tab-1" name="tabular-1" checked>
<div class="tab col-simple">
<div class="col-simple">
<div class="col-xs-12 col-simple">
<section id="mediaFile" class="portlet col-simple">
<div class="portlet-body col-simple">
<iframe class="col-simple" id="iRenderFrame" src="<?= UriFactory::build('{/api}orw/resource/render?id=' . $resource->id); ?>" loading="lazy" allowfullscreen></iframe>
</div>
</section>
</div>
</div>
<div class="col-simple">
<div class="col-xs-12 col-simple">
<section id="mediaFile" class="portlet col-simple">
<div class="portlet-body col-simple">
<iframe class="col-simple" id="iRenderFrame" src="<?= UriFactory::build('{/api}orw/resource/render?id=' . $resource->id); ?>" loading="lazy" allowfullscreen></iframe>
</div>
</section>
</div>
</div>
</div>
<input type="radio" id="resource-c-tab-2" name="tabular-1">
@ -91,7 +102,7 @@ $resource = $this->getData('resource') ?? new \Modules\OnlineResourceWatcher\Mod
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>
</div>