diff --git a/server/CMakeLists.txt b/server/CMakeLists.txt index 8b44543..8ed8dd6 100755 --- a/server/CMakeLists.txt +++ b/server/CMakeLists.txt @@ -2,6 +2,7 @@ cmake_minimum_required(VERSION 3.22) project(OnlineResourceWatcherServerApp VERSION 1.0.0 LANGUAGES CXX) add_executable(OnlineResourceWatcherServerApp main.cpp) +set(CMAKE_BUILD_TYPE "Debug") set(CMAKE_CXX_FLAGS "-march=native -msse2 -mavx -maes") set(CMAKE_CXX_STANDARD 17) diff --git a/server/Controller/ApiController.h b/server/Controller/ApiController.h index f7f16c2..3987fbe 100755 --- a/server/Controller/ApiController.h +++ b/server/Controller/ApiController.h @@ -33,7 +33,7 @@ namespace Controller { void printHelp(int argc, char **argv) { printf(" The Online Resource Watcher app developed by jingga checks online or local resources\n"); - printf(" for changes and and informs the user about them.\n\n"); + printf(" for changes and informs the user about them.\n\n"); printf(" Run: ./App ....\n\n"); printf(" -h: Prints the help output\n"); printf(" -v: Prints the version\n"); @@ -61,12 +61,11 @@ namespace Controller { typedef struct { Models::Resource **resources; int count = 0; - int simultaneous = 0; - } ResourceData; + } ThreadData; void onlineResourceThreaded(void *arg) { - ResourceData *data = (ResourceData *) arg; + ThreadData *data = (ThreadData *) arg; char **urls = (char **) malloc(data->count * sizeof(char *)); int i; @@ -75,11 +74,19 @@ namespace Controller { urls[i] = data->resources[i]->uri; } - Utils::FileUtils::file_body *multi = Utils::WebUtils::multi_download(urls, data->count, data->simultaneous); + Utils::FileUtils::file_body *multi = Utils::WebUtils::multi_download( + urls, + data->count, + 5, + 0, + (ResourceTypes *) {.size = 4, .resources = {"jpg", "png", "gif", "css"}} + ); // @todo: flag for downloading resources types (e.g. js, css, img) // @todo: limit filesize to abort downloading large files - free(urls); + if (urls != NULL) { + free(urls); + } bool hasChanged = false; meow_u128 tempHash; @@ -117,8 +124,13 @@ namespace Controller { Models::free_Resource(data->resources[i]); } - free(data->resources); - free(arg); + if (data->resources != NULL) { + free(data->resources); + } + + if (arg != NULL) { + free(arg); + } } void offlineResourceThreaded(void *arg) @@ -142,12 +154,29 @@ namespace Controller { resources[i].id = atoll(resourceIdStrings[i]); } - free(resourceIdStrings); + if (resourceIdStrings != NULL) { + free(resourceIdStrings); + } } else { - // find and load all relevant ids from the database - resources = (Models::Resource *) DataStorage::Database::DataMapperFactory::get(&Models::ResourceMapper) - ->where((char *) "status", (void *) Models::ResourceStatus::RESOURCE_ACTIVE) - ->execute(); + // @todo: limit memory usage by doing this multiple times in a loop with limits; + DataStorage::Database::QueryResult results = app->db->query_execute( + (char *) "SELECT * from oms.orw_resource WHERE oms.orw_resource_status = 1" + ); + + resources = (Models::Resource *) malloc(results.rows * sizeof(Models::Resource)); + for (size_t row = 0; row < results.rows; ++row) { + resources[row] = {}; + + for (i = 0; i < results.columns; ++i) { + if (results.results[row * results.columns + i] != NULL) { + free(results.results[row * results.columns + i]); + } + } + } + + if (results.results != NULL) { + free(results.results); + } } // How many resources are handled in one thread @@ -174,10 +203,9 @@ namespace Controller { // Handle online resources in batches here: if (j > 0 && (j == THREAD_SIZE || i + 1 >= idLength)) { - ResourceData *data = (ResourceData *) malloc(sizeof(ResourceData)); + ThreadData *data = (ThreadData *) malloc(sizeof(ThreadData)); data->resources = onlineResources; data->count = j; - data->simultaneous = THREAD_SIZE; Threads::pool_add_work(app->pool, onlineResourceThreaded, data); @@ -189,10 +217,9 @@ namespace Controller { // Handle offline resources in batches here: if (k > 0 && (k == THREAD_SIZE || i + 1 >= idLength)) { - ResourceData *data = (ResourceData *) malloc(sizeof(ResourceData)); + ThreadData *data = (ThreadData *) malloc(sizeof(ThreadData)); data->resources = offlineResources; data->count = k; - data->simultaneous = THREAD_SIZE; Threads::pool_add_work(app->pool, offlineResourceThreaded, data); diff --git a/server/bin/OnlineResourceWatcherServerApp b/server/bin/OnlineResourceWatcherServerApp index 86fb14f..a3149a1 100755 Binary files a/server/bin/OnlineResourceWatcherServerApp and b/server/bin/OnlineResourceWatcherServerApp differ diff --git a/server/bin/cmake_install.cmake b/server/bin/cmake_install.cmake old mode 100755 new mode 100644 index 64c5280..a10b330 --- a/server/bin/cmake_install.cmake +++ b/server/bin/cmake_install.cmake @@ -12,7 +12,7 @@ if(NOT DEFINED CMAKE_INSTALL_CONFIG_NAME) string(REGEX REPLACE "^[^A-Za-z0-9_]+" "" CMAKE_INSTALL_CONFIG_NAME "${BUILD_TYPE}") else() - set(CMAKE_INSTALL_CONFIG_NAME "") + set(CMAKE_INSTALL_CONFIG_NAME "Debug") endif() message(STATUS "Install configuration: \"${CMAKE_INSTALL_CONFIG_NAME}\"") endif() diff --git a/server/bin/config.json b/server/bin/config.json index 6691287..3e2d4f8 100755 --- a/server/bin/config.json +++ b/server/bin/config.json @@ -109,6 +109,14 @@ "id": "frontend", "lang": "en" } + }, + "threads": { + "count": 5 + }, + "resources": { + "online": { + "donwloads": 10 + } } }, "language": [ diff --git a/server/cOMS b/server/cOMS index bc465ff..24068a9 160000 --- a/server/cOMS +++ b/server/cOMS @@ -1 +1 @@ -Subproject commit bc465ff0a8ea45ccb42b06fb7edb050bfa523791 +Subproject commit 24068a973461f7d5aecfce6fb4feb3d382ba7140 diff --git a/server/main.cpp b/server/main.cpp index f6bac2e..f19a4f7 100755 --- a/server/main.cpp +++ b/server/main.cpp @@ -10,6 +10,7 @@ #include #include +#include #include "cOMS/Utils/ApplicationUtils.h" #include "cOMS/DataStorage/Database/Connection/ConnectionAbstract.h" @@ -18,6 +19,7 @@ #include "cOMS/Router/Router.h" #include "cOMS/Threads/Thread.h" #include "cOMS/Application/ApplicationAbstract.h" +#include "Models/ResourceMapper.h" #include "Routes.h" @@ -101,6 +103,8 @@ int main(int argc, char **argv) Threads::pool_destroy(app.pool); + DataStorage::Database::free_MapperData((DataStorage::Database::MapperData *) &Models::ResourceMapper); + app.db->close(); app.db = NULL;