cOMS/asset/AssetArchive.h

/**
 * Jingga
 *
 * @copyright Jingga
 * @license    License 2.0
 * @version   1.0.0
 * @link      https://jingga.app
 */
#ifndef COMS_ASSET_ARCHIVE_H
#define COMS_ASSET_ARCHIVE_H

#include "../stdlib/Types.h"
#include "../utils/StringUtils.h"
#include "../utils/EndianUtils.h"
#include "../utils/Utils.h"
#include "../memory/RingMemory.h"
#include "../memory/BufferMemory.h"
#include "../image/Image.cpp"
#include "../image/Qoi.h"
#include "../object/Mesh.h"
#include "../object/Texture.h"
#include "../audio/Audio.cpp"
#include "../audio/Qoa.h"
#include "../font/Font.h"
#include "../localization/Language.h"
#include "../ui/UITheme.h"
#include "AssetManagementSystem.h"
#include "../system/FileUtils.cpp"
#include "../stdlib/Simd.h"
#include "../compiler/CompilerUtils.h"

#define ASSET_ARCHIVE_VERSION 1

struct AssetArchiveElement {
    uint32 type;

    uint32 start;
    uint32 length;
    uint32 uncompressed;

    uint32 dependency_start; // actual index for asset_dependencies
    uint32 dependency_count;
};

// It is important to understand that for performance reasons the assets addresses are stored in an array
// This makes it very fast to access because there is only one indirection.
// On the other hand we can only find assets by their ID/location and not by name.
struct AssetArchiveHeader {
    int32 version;

    uint32 asset_count;
    uint32 asset_dependency_count;

    AssetArchiveElement* asset_element; // is not the owner of the data
    int32* asset_dependencies; // is not the owner of the data
};

struct AssetArchive {
    AssetArchiveHeader header;
    byte* data; // owner of the data

    FileHandle fd;
    FileHandle fd_async;

    // @performance We still need to implement the loading with this and then profile it to see if it is faster.
    // If not remove
    MMFHandle mmf;

    // This is used to tell the asset archive in which AssetManagementSystem (AMS) which asset type is located.
    // Remember, many AMS only contain one asset type (e.g. image, audio, ...)
    byte asset_type_map[ASSET_TYPE_SIZE];
};

// Calculates how large the header memory has to be to hold all its information
int32 asset_archive_header_size(AssetArchive* __restrict archive, const byte* __restrict data)
{
    data += sizeof(archive->header.version);

    int32 asset_count = SWAP_ENDIAN_LITTLE(*((uint32 *) data));
    data += sizeof(archive->header.asset_count);

    int32 asset_dependency_count = SWAP_ENDIAN_LITTLE(*((uint32 *) data));
    data += sizeof(archive->header.asset_dependency_count);

    return sizeof(archive->header.version)
        + sizeof(archive->header.asset_count)
        + sizeof(archive->header.asset_dependency_count)
        + asset_count * sizeof(AssetArchiveElement)
        + asset_dependency_count * sizeof(int32);
}

void asset_archive_header_load(AssetArchiveHeader* __restrict header, const byte* __restrict data, [[maybe_unused]] int32 steps = 8)
{
    header->version = SWAP_ENDIAN_LITTLE(*((int32 *) data));
    data += sizeof(header->version);

    header->asset_count = SWAP_ENDIAN_LITTLE(*((uint32 *) data));
    data += sizeof(header->asset_count);

    header->asset_dependency_count = SWAP_ENDIAN_LITTLE(*((uint32 *) data));
    data += sizeof(header->asset_dependency_count);

    memcpy(header->asset_element, data, header->asset_count * sizeof(AssetArchiveElement));
    data += header->asset_count * sizeof(AssetArchiveElement);

    SWAP_ENDIAN_LITTLE_SIMD(
        (int32 *) header->asset_element,
        (int32 *) header->asset_element,
        header->asset_count * sizeof(AssetArchiveElement) / 4, // everything is 4 bytes -> easy to swap
        steps
    );

    if (header->asset_dependency_count) {
        header->asset_dependencies = (int32 *) ((byte *) header->asset_element + header->asset_count * sizeof(AssetArchiveElement));
    }

    memcpy(header->asset_dependencies, data, header->asset_dependency_count * sizeof(int32));
    SWAP_ENDIAN_LITTLE_SIMD(
        (int32 *) header->asset_dependencies,
        (int32 *) header->asset_dependencies,
        header->asset_count * header->asset_dependency_count, // everything is 4 bytes -> easy to swap
        steps
    );
}

inline
AssetArchiveElement* asset_archive_element_find(const AssetArchive* archive, int32 id)
{
    return &archive->header.asset_element[id];
}

void asset_archive_load(AssetArchive* archive, const char* path, BufferMemory* buf, RingMemory* ring, int32 steps = 8)
{
    PROFILE(PROFILE_ASSET_ARCHIVE_LOAD, path, false, true);

    LOG_1(
        "Load AssetArchive %s",
        {{LOG_DATA_CHAR_STR, (void *) path}}
    );

    archive->fd = file_read_handle(path);
    if (!archive->fd) {
        return;
    }

    archive->fd_async = file_read_async_handle(path);
    if (!archive->fd_async) {
        return;
    }
    archive->mmf = file_mmf_handle(archive->fd_async);

    FileBody file = {};
    file.size = 64;

    // Find header size
    file.content = ring_get_memory(ring, file.size, 4);
    file_read(archive->fd, &file, 0, file.size);
    file.size = asset_archive_header_size(archive, file.content);

    // Reserve memory for the header
    archive->data = buffer_get_memory(
        buf,
        file.size
            - sizeof(archive->header.version)
            - sizeof(archive->header.asset_count)
            - sizeof(archive->header.asset_dependency_count),
        4
    );

    archive->header.asset_element = (AssetArchiveElement *) archive->data;

    // Read entire header
    file.content = ring_get_memory(ring, file.size);
    file_read(archive->fd, &file, 0, file.size);
    asset_archive_header_load(&archive->header, file.content, steps);

    LOG_1(
        "Loaded AssetArchive %s with %d assets",
        {{LOG_DATA_CHAR_STR, (void *) path}, {LOG_DATA_UINT32, (void *) &archive->header.asset_count}}
    );
}

// @question Do we want to allow a callback function?
// Very often we want to do something with the data (e.g. upload it to the gpu)
// Maybe we could just accept a int value which we set atomically as a flag that the asset is complete?
// this way we can check much faster if we can work with this data from the caller?!
// The only problem is that we need to pass the pointer to this int in the thrd_queue since we queue the files to load there
Asset* asset_archive_asset_load(const AssetArchive* archive, int32 id, AssetManagementSystem* ams, RingMemory* ring)
{
    // Create a string representation from the asset id
    // We can't just use the asset id, since an int can have a \0 between high byte and low byte
    // @question We maybe can switch the AMS to work with ints as keys.
    // We would then have to also create an application specific enum for general assets,
    // that are not stored in the asset archive (e.g. color palette, which is generated at runtime).
    char id_str[9];
    int_to_hex(id, id_str);

    PROFILE(PROFILE_ASSET_ARCHIVE_ASSET_LOAD, id_str, false, true);
    // @todo add calculation from element->type to ams index. Probably requires an app specific conversion function

    // We have to mask 0x00FFFFFF since the highest bits define the archive id, not the element id
    AssetArchiveElement* element = &archive->header.asset_element[id & 0x00FFFFFF];

    byte component_id = archive->asset_type_map[element->type];
    //AssetComponent* ac = &ams->asset_components[component_id];

    LOG_2(
        "Load asset %d from archive %d for AMS %d with %n B compressed and %n B uncompressed",
        {{LOG_DATA_UINT64, &id}, {LOG_DATA_UINT32, &element->type}, {LOG_DATA_BYTE, &component_id}, {LOG_DATA_UINT32, &element->length}, {LOG_DATA_UINT32, &element->uncompressed}}
    );

    Asset* asset = thrd_ams_get_asset_wait(ams, id_str);

    if (asset) {
        // Prevent garbage collection
        asset->state &= ~ASSET_STATE_RAM_GC;
        asset->state &= ~ASSET_STATE_VRAM_GC;

        return asset;
    }

    // @bug Couldn't the asset become available from thrd_ams_get_asset_wait to here?
    // This would mean we are overwriting it
    // A solution could be a function called thrd_ams_get_reserve_wait() that reserves, if not available
    // However, that function would have to lock the ams during that entire time

    if (element->type == 0) {
        asset = thrd_ams_reserve_asset(ams, (byte) component_id, id_str, element->uncompressed);
        asset->official_id = id;
        asset->ram_size = element->uncompressed;

        FileBody file = {};
        file.content = asset->self;

        // @performance Consider to implement general purpose fast compression algorithm

        // We are directly reading into the correct destination
        file_read(archive->fd, &file, element->start, element->length);
    } else {
        // @performance In this case we may want to check if memory mapped regions are better.
        // 1. I don't think they work together with async loading
        // 2. Profile which one is faster
        // 3. The big benefit of mmf would be that we can avoid one memcpy and directly load the data into the object
        // 4. Of course the disadvantage would be to no longer have async loading

        // We are reading into temp memory since we have to perform transformations on the data
        FileBodyAsync file = {};
        file_read_async(archive->fd_async, &file, element->start, element->length, ring);

        // This happens while the file system loads the data
        // The important part is to reserve the uncompressed file size, not the compressed one
        asset = thrd_ams_reserve_asset(ams, (byte) component_id, id_str, element->uncompressed);
        asset->official_id = id;

        asset->state |= ASSET_STATE_IN_RAM;

        file_async_wait(archive->fd_async, &file.ov, true);
        switch (element->type) {
            case ASSET_TYPE_IMAGE: {
                // @todo Do we really want to store textures in the asset management system or only images?
                // If it is only images then we need to somehow also manage textures
                Texture* texture = (Texture *) asset->self;
                texture->image.pixels = (byte *) (texture + 1);

                qoi_decode(file.content, &texture->image);

                asset->vram_size = texture->image.pixel_count * image_pixel_size_from_type(texture->image.image_settings);
                asset->ram_size = asset->vram_size + sizeof(Texture);

                #if OPENGL || VULKAN
                    // If opengl, we always flip
                    if (!(texture->image.image_settings & IMAGE_SETTING_BOTTOM_TO_TOP)) {
                        image_flip_vertical(ring, &texture->image);
                    }
                #endif
            } break;
            case ASSET_TYPE_AUDIO: {
                Audio* audio = (Audio *) asset->self;
                audio->data = (byte *) (audio + 1);

                qoa_decode(file.content, audio);
            } break;
            case ASSET_TYPE_OBJ: {
                Mesh* mesh = (Mesh *) asset->self;
                mesh->data = (byte *) (mesh + 1);

                mesh_from_data(file.content, mesh);
            } break;
            case ASSET_TYPE_LANGUAGE: {
                Language* language = (Language *) asset->self;
                language->data = (byte *) (language + 1);

                language_from_data(file.content, language);
            } break;
            case ASSET_TYPE_FONT: {
                Font* font = (Font *) asset->self;
                font->glyphs = (Glyph *) (font + 1);

                font_from_data(file.content, font);
            } break;
            case ASSET_TYPE_THEME: {
                UIThemeStyle* theme = (UIThemeStyle *) asset->self;
                theme->data = (byte *) (theme + 1);

                theme_from_data(file.content, theme);
            } break;
            default: {
                UNREACHABLE();
            }
        }
    }

    // Even though dependencies are still being loaded
    // the main program should still be able to do some work if possible
    thrd_ams_set_loaded(asset);

    LOG_2(
        "Loaded asset %d from archive %d for AMS %d with %n B compressed and %n B uncompressed",
        {{LOG_DATA_UINT64, &id}, {LOG_DATA_UINT32, &element->type}, {LOG_DATA_BYTE, &component_id}, {LOG_DATA_UINT32, &element->length}, {LOG_DATA_UINT32, &element->uncompressed}}
    );

    // @performance maybe do in worker threads? This just feels very slow
    // @bug dependencies might be stored in different archives?
    for (uint32 i = 0; i < element->dependency_count; ++i) {
        asset_archive_asset_load(archive, id, ams, ring);
    }

    return asset;
}

#endif