/**
 * This file is part of the Amalthea library.
 * Copyright (C) 2019-2024 Eugene 'Vindex' Stulin
 * Distributed under the BSL 1.0 or (at your option) the GNU LGPL 3.0 or later.
 *
 * The module implements recognition of file formats.
 * Main function of this module:
 *     `FileFormat getFileFormat(string filepath)`
 * Recognition is carried out according to a complex algorithm based on
 * a special file containing information about file formats.
 * The default path of such file for a normal library installation is:
 *     `/etc/amalthea-${DC}/fileformats.json`
 * You can expand the search paths for format descriptions with:
 *     `void addNewPathToFindFileFormats(string dir)`
 */

module amalthea.fileformats;

import
    std.algorithm,
    std.array,
    std.datetime,
    std.file,
    std.format,
    std.json,
    std.path,
    std.process,
    std.stdio,
    std.string,
    std.typecons,
    std.utf;

import amalthea.libcore;
import amalthea.dataprocessing : calcIntersection, copyArray;
import amalthea.fs : getRegularFiles, isSymlinkToDir;

pragma(lib, "gio-2.0");
pragma(lib, "glib-2.0");


struct Signature {
    /// Key byte sequence.
    ubyte[] bytes;
    /// Offset of the signature from the beginning of the file.
    uint offset;

    this(ref return scope const Signature rhs) {
        this.bytes = rhs.bytes.dup;
        this.offset = rhs.offset;
    }
}


/*******************************************************************************
 * File format info.
 */
struct FileFormat {
    /// Format name.
    string format;
    /// Key byte sequences and offsets.
    Signature[] signatures;
    /// List of file suffixes used by the format.
    string[] extensions;
    /// Format brief description.
    string description;
    /// Big description. Reserved for future use.
    string extendedDescription;
    /// Format group name (Audio, Pictures, Archives, etc.).
    string group;
    /** If true, then the format is valid only
     *  if both the extension and the signature match.
     */
    bool completeMatchOnly;

    /// Implements comparison.
    int opCmp(in FileFormat rhs) const {
        if (this.format > rhs.format) {
            return 1;
        }
        if (this.format < rhs.format) {
            return -1;
        }
        return 0;
    }

    private void copy(ref scope const FileFormat rhs) {
        this.format = rhs.format.idup;
        this.signatures = copyArray(rhs.signatures);
        this.extensions = rhs.extensions.dup;
        this.description = rhs.description.dup;
        this.extendedDescription = rhs.extendedDescription.dup;
        this.group = rhs.group.idup;
        this.completeMatchOnly = rhs.completeMatchOnly;
    }

    this(ref return scope const FileFormat rhs) {
        copy(rhs);
    }

    ref FileFormat opAssign(ref scope const FileFormat rhs) {
        copy(rhs);
        return this;
    }
}


/*******************************************************************************
 * The function gets the FileFormat structure with info about file format.
 *
 * Params:
 *     filepath = Path to required file.
 *     additExt = File extension as hint for recognition algorithm.
 *                Not needed in most cases. Can be used if for some reason
 *                the file name does not have an extension.
 *
 * Returns: a FileFormat object containing some format information.
 */
FileFormat getFileFormat(string filepath, string additExt) {
    if (!filepath.exists || !filepath.isFile) {
        throw new FileException("File is not regular or not readable.");
    }
    return signatureSearch(filepath, additExt);
}


/*******************************************************************************
 * The function gets the FileFormat structure with info about file format.
 *
 * Params:
 *     filepath = Path to required file.
 *
 * Returns: a FileFormat object containing some format information.
 */
FileFormat getFileFormat(string filepath) {
    return getFileFormat(filepath, "");
}


/*******************************************************************************
 * The function returns true if specified file contains plain Unicode-text.
 */
bool isUnicodeTextFile(string filepath) nothrow {
    try {
        File(filepath, "r").byLine.each!validate;
    } catch (Exception e) {
        return false;
    }
    return true;
}


private FileFormat[] searchByExtHint(
    const FileFormat[] candidates, string additExt
) {
    additExt = additExt.toLower;
    FileFormat[] newCandidatesWithExtensions;
    FileFormat[] newCandidatesWithoutExtensions;
    foreach(ftype; candidates) {
        foreach(ext; ftype.extensions) {
            if (ext == "*") {
                newCandidatesWithoutExtensions ~= cast(FileFormat)ftype;
            } else if (additExt == ext) {
                newCandidatesWithExtensions ~= cast(FileFormat)ftype;
            }
        }
    }
    return newCandidatesWithoutExtensions ~ newCandidatesWithExtensions;
}


private FileFormat[] searchByExtensions(
    const FileFormat[] candidates, string fpath
) {
    fpath = fpath.toLower;
    FileFormat[] newCandidatesWithExtensions;
    FileFormat[] newCandidatesWithoutExtensions;
    foreach(ftype; candidates) {
        foreach(ext; ftype.extensions) {
            auto end = "." ~ ext;
            if (ext == "*") {
                newCandidatesWithoutExtensions ~= cast(FileFormat)ftype;
            } else if (fpath.endsWith(end) || fpath.canFind(end ~ ".")) {
                newCandidatesWithExtensions ~= cast(FileFormat)ftype;
            }
        }
    }
    return newCandidatesWithoutExtensions ~ newCandidatesWithExtensions;
}


static FileFormat[] searchBySignature(
    const FileFormat[] candidates,
    const ubyte[] data
) {
    FileFormat[] newCandidates;
    foreach(ftype; candidates) {
        if (ftype.signatures.length == 0) {
            newCandidates ~= cast(FileFormat)ftype;
        }
        foreach(s; ftype.signatures) {
            auto off = s.offset;
            auto signature = s.bytes;
            if (signature.length + off > data.length) {
                continue;
            }
            if (signature.empty) {
                newCandidates ~= cast(FileFormat)ftype;
                break;
            } else if (signature == data[off .. off + signature.length]) {
                newCandidates ~= cast(FileFormat)ftype;
                break;
            }
        }
    }
    return newCandidates;
}


private FileFormat identifyLikelyFormat(
    const FileFormat[] candidates,
    string filepath
)
in {
    assert(candidates.length > 1);
}
do {
    size_t getMaxNumberOfBytes(const Signature[] signatures) {
        size_t maxNumber;
        foreach(s; signatures) {
            if (s.bytes.length > maxNumber) {
                maxNumber = s.bytes.length;
            }
        }
        return maxNumber;
    }
    FileFormat result = candidates[0];
    FileFormat[] newCandidates = [result];
    size_t maxNumberOfBytes = getMaxNumberOfBytes(result.signatures);
    size_t currentFormatMaxSignatureLength;
    foreach(f; candidates[1 .. $]) {
        currentFormatMaxSignatureLength = getMaxNumberOfBytes(f.signatures);
        if (currentFormatMaxSignatureLength > maxNumberOfBytes) {
            maxNumberOfBytes = currentFormatMaxSignatureLength;
            result = f;
            newCandidates = [result];
        } else if (currentFormatMaxSignatureLength == maxNumberOfBytes) {
            result = f;
            newCandidates ~= result;
        }
    }
    // candidates with the same signature lengths
    if (newCandidates.length > 1) {
        size_t currentMaxExtensionLength;
        foreach(ftype; newCandidates) {
            foreach(ext; ftype.extensions) {
                auto end = "." ~ ext;
                if (filepath.endsWith(end)) {
                    if (currentMaxExtensionLength < ext.length) {
                        currentMaxExtensionLength = ext.length;
                        result = ftype;
                    }
                }
            }
        }
    }
    return result;
}


private bool identifyFacelessFile(string filepath) {
    if (std.path.extension(filepath) == "" && std.file.getSize(filepath) == 0) {
        return true;
    }
    return false;
}


private FileFormat getPlainTextFormat() {
    return FileFormat("TXT", null, null, "Plain text document", "", "Text");
}


private FileFormat signatureSearch(string filepath, string additExt = "") {
    auto f = File(filepath, "r");
    ubyte[] buffer;
    buffer.length = 1024;
    ubyte[] data = f.rawRead(buffer);  // todo: mode with O_NOATIME

    FileFormat[] signCandidates = searchBySignature(fileformats, data);
    FileFormat[] extCandidates = additExt.empty ?
        searchByExtensions(fileformats, filepath) :
        searchByExtHint(fileformats, additExt);
    FileFormat[] candidates = calcIntersection(signCandidates, extCandidates);
    if (candidates.empty) {
        signCandidates = signCandidates
            .filter!(c => !c.signatures.empty && !c.completeMatchOnly)
            .array;
        if (signCandidates.empty) {
            return altFormatSearch(filepath);
        } else if (signCandidates.length == 1) {
            return signCandidates[0];
        }
        return identifyLikelyFormat(signCandidates, filepath);
    }
    if (candidates.length == 1) {
        return candidates[0];
    }
    // where candidates.length > 1
    return identifyLikelyFormat(candidates, filepath);
}


extern(C)
static char* g_content_type_guess(
    const char* filename,
    const byte* data,
    size_t data_size,
    bool* result_uncertain
);

extern(C)
static void g_free(void* mem);

extern(C)
char* g_content_type_get_mime_type(const char* type);

extern(C)
char* g_content_type_get_description(const char* type);


/// Returns tuple with MIME type and its description.
auto contentTypeGuess(alias useEnglish=true)(string filepath) {
    static if (useEnglish) {
        auto lang = environment.get("LANG", "");
        environment["LANG"] = "C";
        scope(exit) environment["LANG"] = lang;
    }

    auto raw = cast(byte[])read(filepath, 1024);
    byte* data = raw.ptr;
    char* type = g_content_type_guess(
        filepath.toStringz, data, raw.length, null
    );
    string mimetype = type.fromStringz.idup;
    char* descr = g_content_type_get_description(type);
    string description = descr.fromStringz.idup;
    g_free(type);
    g_free(descr);
    return tuple(mimetype, description);
}


private FileFormat altFormatSearch(string filepath) {
    if (identifyFacelessFile(filepath)) {
        return getPlainTextFormat();
    }
    string ext = std.path.extension(filepath).stripLeft('.');
    auto mimeAndDescription = contentTypeGuess(filepath);
    string mimetype = mimeAndDescription[0];
    string mimegroup = mimetype.split('/')[0];
    string mimeformat = mimetype.split('/')[1];
    string description = mimeAndDescription[1];

    FileFormat ff;
    ff.format = ext.empty ? mimeformat : ext.toUpper;
    ff.description = description.capitalize;
    if (ff.description == "Plain text document" && ff.format == "plain") {
        ff.format = "TXT";
    } else if (mimeformat.endsWith("src")) {
        ff.format = ff.description;
    }
    if (isUnicodeTextFile(filepath) || mimegroup == "text") {
        ff.group = "Text";
    }
    return ff;
}


/*******************************************************************************
 * The functions allows to extend paths to find JSON files with file formats.
 *
 * Params:
 *     dir = Directory to search for JSON files.
 */
void addNewPathToFindFileFormats(string dir) {
    if (!exists(dir) || !isDir(dir) || !isSymlinkToDir(dir)) {
        throw new FileException(dir, ": directory not found.");
    }
    auto entries = getRegularFiles(dir, No.hidden)
        .filter!(a => a.path.extension.toLower == ".json");
    JSONValue[] jsonFormatArray;
    foreach(entry; entries) {
        jsonFormatArray ~= entry.path.readText.parseJSON.array;
    }
    extendFileFormatBase(jsonFormatArray);
}


immutable string canonBase;
shared static this() {
    enum fileformatsJSONText = import("res/fileformats.json");
    canonBase = format!"/etc/amalthea-%s/fileformats.json"(amaltheaCompiler);
    JSONValue[] jsonFormatArray;
    if (exists(canonBase)) {
        string text = readText(canonBase);
        jsonFormatArray = parseJSON(text).array;
    } else {
        jsonFormatArray = parseJSON(fileformatsJSONText).array;
    }
    extendFileFormatBase(jsonFormatArray);
}


private __gshared FileFormat[] fileformats;

/// Returns array of supported file formats.
FileFormat[] getAllFileFormats() {
    return fileformats.dup;
}


private void extendFileFormatBase(JSONValue[] jsonFormatArray) {
    auto spec = singleSpec("%x");
    foreach(j; jsonFormatArray) {
        FileFormat newFormat;
        newFormat.format = j["format"].str;
        foreach(s; j["signatures"].array) {
            Signature newSignature;
            newSignature.offset = cast(uint)s["offset"].integer;
            foreach(strByte; s["hex_signature"].str.split(' ')) {
                newSignature.bytes ~= unformatValue!ubyte(strByte, spec);
            }
            newFormat.signatures ~= newSignature;
        }
        foreach(ext; j["extensions"].array) {
            newFormat.extensions ~= ext.str;
        }
        newFormat.description = j["description"].str;
        if ("extended_description" in j) {
            newFormat.extendedDescription = j["extended_description"].str;
        }
        newFormat.group = j["group"].str;
        fileformats ~= newFormat;  // global variable
    }
}