/**
 * This file is part of the Amalthea library.
 * Copyright (C) 2018-2024 Eugene 'Vindex' Stulin
 * Distributed under the BSL 1.0 or (at your option) the GNU LGPL 3.0 or later.
 *
 * The module contains some useful functions for data processing,
 * including functions for working with dynamic arrays and associative arrays,
 * functions for working with command line arguments,
 * operations on arrays as sets,
 * some functions for working with strings, etc.
 */

module amalthea.dataprocessing;

public import amalthea.libcore;

import std.algorithm, std.array, std.json;
import std.format : formattedWrite;
import std.range.primitives : isInputRange;
static import std.traits;


/*******************************************************************************
 * Implementation of the intersection operation for arrays.
 */
T calcIntersection(T)(const T[] arrays ...) {
    import std.algorithm.setops : setIntersection;
    if (arrays.length == 1) return copyArray(arrays[0]);
    T result = setIntersection(
        sort(copyArray(arrays[0])), sort(copyArray(arrays[1]))
    ).array;
    for (auto i = 2; i < arrays.length; ++i) {
        result = setIntersection(result, sort(copyArray(arrays[i]))).array;
    }
    return result;
}
///
unittest {
    int[] arr = calcIntersection([3, 4, 1, -15], [9, 3, 1], [1, 99, 3]);
    assert(arr == [1, 3]);
}


/*******************************************************************************
 * Implementation of the union operation for arrays.
 */
T calcUnion(T)(const T[] arrays ...) {
    if (arrays.length == 1) {
        return copyArray(arrays[0]);
    }
    T result = copyArray(arrays[0]);
    for (size_t i = 1; i < arrays.length; ++i) {
        foreach(el; copyArray(arrays[i])) {
            if (!canFind(result, el)) {
                result ~= el;
            }
        }
    }
    return std.algorithm.sort(result).array;
}
///
unittest {
    int[] arr = calcUnion([3, 4, 1, -15], [9, 3, 1], [1, 99, 3]);
    assert(arr == [-15, 1, 3, 4, 9, 99]);
}


/*******************************************************************************
 * Implementation of the complement operation for arrays.
 */
T calcComplement(T)(const T[] arrays ...) {
    if (arrays.length == 1) {
        return arrays[0].copyArray;
    }
    T result = setDifference(
        arrays[0].copyArray.sort,
        arrays[1].copyArray.sort
    ).array;
    for (auto i = 2; i < arrays.length; ++i) {
        result = array(setDifference(result, arrays[i].copyArray.sort));
    }
    return result;
}
///
unittest {
    int[] arr = calcComplement([3, 4, 1, -15], [9, 3, 1]);
    assert(arr == [-15, 4]);
}


/*******************************************************************************
 * The function removes consecutive string duplicates.
 */
string removeDuplicateConsecutiveSubstring(string s, string substring) {
    while(s.canFind(substring~substring)) {
        s = s.replace(substring~substring, substring);
    }
    return s;
}
///
unittest {
    string s = removeDuplicateConsecutiveSubstring("AAAAbooeAAAuuvAww", "A");
    assert(s == "AbooeAuuvAww");
    s = removeDuplicateConsecutiveSubstring("AAAAbcooeAAAgguvAww", "AA");
    assert(s == "AAbcooeAAAgguvAww");
}


/*******************************************************************************
 * The function removes duplicate elements from an array.
 */
ref T[] removeDuplicate(T)(ref T[] arr) {
    if (arr.empty) return arr;
    for (auto i = arr.length-1; i > 0; --i) {
        if (arr[0 .. i].canFind(arr[i])) {
            arr = arr.remove(i);
        }
    }
    return arr;
}
///
unittest {
    int[] values = [1, 3, 1, 7, 2, 19, 5, 2, 2];
    removeDuplicate(values);
    assert(values == [1, 3, 7, 2, 19, 5]);
}


/*******************************************************************************
 * Returns array of tuples with pairs of keys and values from associative array.
 */
auto assocArrayItems(V, K)(V[K] aarray) {
    Tuple!(K, V)[] pairs;
    foreach(item; aarray.byPair) {
        pairs ~= item;
    }
    return pairs;
}


/*******************************************************************************
 * Makes key-sorted array of tuples with pairs of keys and values
 * from associative array.
 * Params:
 *     aarray = Associative array.
 *     desc = Flag for sorting (descending/ascending). By default, No.desc.
 * Returns: Key-sorted array of tuples with elements of the associative array.
 */
auto sortAssocArrayItemsByKeys(V, K)(V[K] aarray, Flag!"desc" desc = No.desc) {
    if (!desc) {
        return aarray.assocArrayItems.sort!((a, b) => a[0] < b[0]).array;
    }
    return aarray.assocArrayItems.sort!((a, b) => a[0] > b[0]).array;
}
unittest {
    int[string] aarray = ["x": 9, "y": 7];
    auto expected = [tuple("x", 9), tuple("y", 7)];
    assert(expected == sortAssocArrayItemsByKeys(aarray));
    expected = [tuple("y", 7), tuple("x", 9)];
    assert(expected == sortAssocArrayItemsByKeys(aarray, Yes.desc));
}


/*******************************************************************************
 * Makes value-sorted array of tuples with pairs of keys and values
 * from associative array.
 * Params:
 *     aarray = Associative array.
 *     desc = Flag for sorting (descending/ascending). By default, No.desc.
 * Returns: Value-sorted array of tuples with elements of the associative array.
 */
auto sortAssocArrayItemsByValues(V, K)(
    V[K] aarray,Flag!"desc" desc = No.desc
) {
    if (!desc) {
        return aarray.assocArrayItems.sort!((a, b) => a[1] < b[1]).array;
    }
    return aarray.assocArrayItems.sort!((a, b) => a[1] > b[1]).array;
}
unittest {
    int[string] aarray = ["x": 9, "y": 7];
    auto expected = [tuple("y", 7), tuple("x", 9)];
    assert(expected == sortAssocArrayItemsByValues(aarray));
    expected = [tuple("x", 9), tuple("y", 7)];
    assert(expected == sortAssocArrayItemsByValues(aarray, Yes.desc));
}


/*******************************************************************************
 * The function arranges the associative array
 * by the required sorting method (string "by" is equal to "values" or "keys"),
 * returns an array of keys and an array of values.
 */
deprecated("Use sortAssocArrayItemsByKeys and sortAssocArrayItemsByValues")
void orderAssociativeArray(alias by, V, K)(
    in V[K] aarray, out K[] orderedKeys, out V[] orderedValues
)
if (by == "values" || by == "keys") {
    if (aarray.length == 0) {
        return;
    }
    if (by == "keys") {
        orderedKeys = aarray.keys.dup;
        orderedKeys.sort;
        orderedValues.length = orderedKeys.length;
        foreach(i, k; orderedKeys) {
            orderedValues[i] = aarray[k];
        }
    } else {
        orderedValues = aarray.values.dup;
        orderedValues.sort;
        orderedKeys.length = orderedValues.length;
        size_t[] passedIndexes;
        foreach(k; aarray.keys) {
            foreach(i, v; orderedValues) {
                if (passedIndexes.canFind(i)) {
                    continue;
                }
                if (aarray[k] == v) {
                    orderedKeys[i] = k;
                    passedIndexes ~= i;
                    break;
                }
            }
        }
    }
}


/*******************************************************************************
 * The function arranges the associative array by keys,
 * returns an array of keys and an array of values.
 */
deprecated("Use sortAssocArrayItemsByKeys")
void orderAssociativeArrayByKeys(V, K)(
    in V[K] aarray, out K[] orderedKeys, out V[] orderedValues
) {
    orderAssociativeArray!"keys"(aarray, orderedKeys, orderedValues);
}


/*******************************************************************************
 * The function arranges the associative array by values,
 * returns an array of keys and an array of values.
 */
deprecated("sortAssocArrayItemsByValues")
void orderAssociativeArrayByValues(V, K)(
    in V[K] aarray, out K[] orderedKeys, out V[] orderedValues
) {
    orderAssociativeArray!"values"(aarray, orderedKeys, orderedValues);
}


/*******************************************************************************
 * This function combines two associative arrays, returns new associative array.
 * If two arrays have the same keys, the values from the first array are used.
 */
V[K] mixAssociativeArrays(K,V)(V[K] firstArray, V[K] secondArray) {
    V[K] array = firstArray.dup;
    foreach(k, v; secondArray) {
        if (k !in array) {
            array[k] = v;
        }
    }
    return array;
}


/*******************************************************************************
 * Function to get a value by parameter.
 * Used to process command line arguments.
 * Params:
 *     args = Command line arguments.
 *     flag = Parameter with "--".
 *     altFlag = Synonym for the flat. By default, empty.
 * Returns: The value of the required parameter.
 */
string getOptionValue(string[] args, string flag, string altFlag = "") {
    auto tmpArgs = args.dup;
    return extractValueForFlag(tmpArgs, flag, altFlag);
}
alias getValueForFlag = getOptionValue;


/*******************************************************************************
 * Function for extracting of value by flag.
 * Unlike getOptionValue(), this function changes the passed `args`,
 * removing the flag with a value.
 * Params:
 *     args = Command line arguments. Changes while the function is running.
 *     flag = Parameter with "--".
 *     altFlag = Synonym for the flat. By default, empty.
 * Returns: The value of the required parameter.
 */
string extractOptionValue(ref string[] args, string flag, string altFlag = "") {
    string value;
    ssize_t index1 = -1;

    cycle1: foreach(option; [flag, altFlag]) {
        foreach(i, arg; args) {
            if (arg == option || arg.startsWith(option~"=")) {
                index1 = i;
                flag = option;
                break cycle1;
            }
        }
    }
    if (index1 == -1) return "";

    if (args[index1].startsWith(flag~"=")) {
        value = args[index1][(flag~"=").length .. $];
        auto tempArgs = args[0 .. index1];
        if (args.length > index1+1) tempArgs ~= args[index1+1 .. $];
        args = tempArgs;
    } else {
        if (args.length < index1+2) return "";
        value = args[index1+1];
        auto tempArgs = args[0 .. index1];
        if (args.length > index1+2) tempArgs ~= args[index1+2 .. $];
        args = tempArgs;
    }
    return value;
}
alias extractValueForFlag = extractOptionValue;

///
unittest {
    string[] args = "app -a value1 value2 -b parameter --nothing".split;
    assert("parameter" == args.getOptionValue("-b"));
    assert("value1" == args.getOptionValue("-a"));
    assert("" == args.getOptionValue("--nothing"));

    assert(args == "app -a value1 value2 -b parameter --nothing".split);

    assert("parameter" == args.extractValueForFlag("-b"));
    assert("app -a value1 value2 --nothing".split == args);
    assert("value1" == args.extractValueForFlag("-a"));
    assert("app value2 --nothing".split == args);

    args = "-a value1 value2 -b parameter --nothing".split;
    assert("parameter" == args.getOptionValue("-b"));
    assert("value1" == args.getOptionValue("-a"));
    assert("" == args.getOptionValue("--nothing"));
    assert("parameter" == args.extractValueForFlag("-b"));
    assert("-a value1 value2 --nothing".split == args);
    assert("value1" == args.extractValueForFlag("-a"));
    assert("value2 --nothing".split == args);

    args = `build --file=/home/user/Project_1/Makefile`.split;
    assert("/home/user/Project_1/Makefile" == args.getOptionValue("--file"));

}


/*******************************************************************************
 * Function to get values as array by flag.
 * Params:
 *     args = Command line arguments.
 *     flag = Parameter with "--".
 *     altFlag = Synonym for the flat. By default, empty.
 * Returns: array of values after the flag.
 */
string[] getOptionRange(string[] args, string flag, string altFlag = "") {
    auto argsCopy = args.dup;
    return extractOptionRange(argsCopy, flag, altFlag);
}


/*******************************************************************************
 * Function for extracting of value as array by flag.
 * Unlike getOptionRange(), this function changes the passed arguments (args),
 * removing the flag with a value.
 * Params:
 *     args = Command line arguments. Changes while the function is running.
 *     flag = Parameter with "--".
 *     altFlag = Synonym for the flat. By default, empty.
 * Returns: array of values after the flag.
 */
string[] extractOptionRange(ref string[] args,
                            string flag,
                            string altFlag = "") {
    string[] range;
    ssize_t index1 = args.countUntil(flag);
    ssize_t index2 = -1;

    if (index1 == -1) index1 = args.countUntil(altFlag);
    if (index1 == -1) return range;
    if (index1 != args.length-1 && !args[index1+1].startsWith("-")) {
        foreach(i, arg; args[index1+1 .. $]) {
            if (arg.startsWith("-")) {
                index2 = index1 + i;
                break;
            }
        }
        if (index2 != -1) {
            range = args[index1+1 .. index2+1];
            args = args[0 .. index1] ~ args[index2+1 .. $];
        } else {
            index2 = args.length - 1;
            range = args[index1+1 .. index2+1];
            args = args[0 .. index1];
        }
    }
    return range;
}


///
unittest {
    string[] args = "app -a value1 value2 -b parameter --nothing".split;
    assert(["value1", "value2"] == getOptionRange(args, "-a"));
    assert(["parameter"] == getOptionRange(args, "-b"));
    string[] nothing = getOptionRange(args, "--nothing");
    assert(nothing.empty);

    args = "app --alt value1 value2 -b parameter --nothing".split;
    assert(["value1", "value2"] == getOptionRange(args, "-a", "--alt"));

    args = "app -a -b thing".split;
    string[] vacuum = getOptionRange(args, "-a");
    assert(vacuum.empty);

    args = "app -a value1 value2 -b parameter --nothing".split;
    assert("value1 value2".split == extractOptionRange(args, "-a"));
    assert("app -b parameter --nothing".split == args);

    args = "-a value1 value2 -b parameter --nothing".split;
    assert("value1 value2".split == extractOptionRange(args, "-a"));
    assert("-b parameter --nothing".split == args);
}


/*******************************************************************************
 * The function converts integer number into an octal form, returns as a string.
 */
string getOctalForm(ulong value) {
    auto writer = std.array.appender!string();
    formattedWrite(writer, "%o", value);
    return writer.data;
}
alias toOctalString = getOctalForm;
///
unittest {
    short eight = 8;
    assert(getOctalForm(eight) == "10");
    assert(33261.getOctalForm == "100755");
}


/*******************************************************************************
 * The function converts integer number into an binary form.
 */
string getBinaryForm(ulong value) {
    auto writer = std.array.appender!string();
    formattedWrite(writer, "%b", value);
    return writer.data;
}


/*******************************************************************************
 * The function returns the index of the array element by value.
 * Params:
 *     haystack = The array in which the search is performed.
 *     needle = Element value in the array.
 * Returns: The index of the array element. -1 if the element was not found.
 */
ssize_t getIndex(T)(T[] haystack, T needle) {
    foreach(i, el; haystack) {
        if (el == needle) {
            return i;
        }
    }
    return -1;
}


/*******************************************************************************
 * The function returns an array without unnecessary elements.
 */
T[] removeElementsByContent(T)(T[] haystack, T needle) {
    for(ssize_t i; i < haystack.length; ++i) {
        if (haystack[i] == needle) {
            if (haystack.length-1 != i) {
                haystack = haystack[0 .. i] ~ haystack[i+1 .. $];
            } else {
                haystack = haystack[0 .. i];
            }
            --i;
        }
    }
    return haystack.dup;
}
///
unittest {
    auto arr = [1, 8, 17, 4, 23, 8, 8, 3, 13, 19, 8];
    auto res = arr.removeElementsByContent(8);
    assert(res == [1, 17, 4, 23, 3, 13, 19]);

    auto lines = ["", "straw", "", "", "one more straw", "", ""];
    auto straws = lines.removeElementsByContent("");
    assert(straws == ["straw", "one more straw"]);
}


/*******************************************************************************
 * Encode associative array using www-form-urlencoding (copied from std.uri).
 * Params:
 *     values = An associative array containing the values to be encoded.
 * Returns: A string encoded using www-form-urlencoding.
 */
string urlEncode(in string[string] values) {
    import std.uri : encodeComponent;
    import std.array : Appender;
    import std.format : formattedWrite;
    if (values.length == 0) return "";
    Appender!string enc;
    enc.reserve(values.length * 128);
    bool first = true;
    foreach (k, v; values) {
        if (!first) enc.put('&');
        formattedWrite(enc, "%s=%s", encodeComponent(k), encodeComponent(v));
        first = false;
    }
    return enc.data;
}


/*******************************************************************************
 * The function converts JSONValue to double.
 */
double convJSONValueToDouble(std.json.JSONValue json) {
    double number;
    static if (__traits(compiles, JSONType.float_)) {
        switch(json.type) {
            case JSONType.float_:   number = json.floating;            break;
            case JSONType.integer:  number = to!double(json.integer);  break;
            case JSONType.uinteger: number = to!double(json.uinteger); break;
            case JSONType..string:   number = to!double(json.str);      break;
            default: break;
        }
    } else {
        switch(json.type) {
            case JSON_TYPE.FLOAT:    number = json.floating;            break;
            case JSON_TYPE.INTEGER:  number = to!double(json.integer);  break;
            case JSON_TYPE.UINTEGER: number = to!double(json.uinteger); break;
            case JSON_TYPE.STRING:   number = to!double(json.str);      break;
            default: break;
        }
    }
    return number;
}


/*******************************************************************************
 * The function similar to functions "canFind" and "among".
 */
bool isAmong(alias pred = (a, b) => a == b, Value, Values...)
            (Value value, Values values)
if (
    Values.length != 0
) {
    foreach (ref v; values) {
        import std.functional : binaryFun;
        if (binaryFun!pred(value, v)) return true;
    }
    return false;
}
/// ditto
bool isAmong(alias pred = (a, b) => a == b, T)
            (T value, const T[] values) {
    foreach (ref v; values) {
        import std.functional : binaryFun;
        if (binaryFun!pred(value, v)) return true;
    }
    return false;
}


/*******************************************************************************
 * Creates new array with specified content.
 */
T[] makeFilledArray(T)(size_t len, T filler) {
    T[] arr = new T[len];
    std.algorithm.fill(arr, filler);
    return arr;
}
///
unittest {
    assert(makeFilledArray(4, 5) == [5, 5, 5, 5]);
    assert(makeFilledArray(10, ' ') == "          ");
}


/*******************************************************************************
 * Cuts off consecutive unnecessary characters from the left side.
 * Returns: truncated string.
 */
string stripLeft(string line, in char symbol=' ') nothrow {
    while (!line.empty && line[0] == symbol) {
        line = line[1 .. $];
    }
    return line;
}
unittest {
    assert(stripLeft("   string ") == "string ");
    assert(stripLeft("ssstring", 's') == "tring");
}


/// ditto
string stripLeft(string line, in char[] symbols) nothrow {
    foreach(char ch; line.dup) {
        if (ch.isAmong(symbols)) {
            line = line[1 .. $];
        } else {
            break;
        }
    }
    return line;
}
unittest {
    auto str = " \t\t string\t";
    auto spaces = [' ', '\t'];
    auto res = "string\t";
    assert(stripLeft(str, spaces) == res);
}


/*******************************************************************************
 * Cuts off consecutive unnecessary characters from the right side.
 * Returns: truncated string.
 */
string stripRight(string line, in char symbol=' ') nothrow {
    while (!line.empty && line[$-1] == symbol) {
        line = line[0 .. $-1];
    }
    return line;
}
unittest {
    assert(stripRight("   string ") == "   string");
    assert(stripRight("ssstringgg", 'g') == "ssstrin");
}


/// ditto
string stripRight(string line, in char[] symbols) nothrow {
    foreach_reverse(char ch; line.dup) {
        if (ch.isAmong(symbols)) {
            line = line[0 .. $-1];
        } else {
            break;
        }
    }
    return line;
}
unittest {
    auto str = " \t\t string\t \t";
    auto spaces = [' ', '\t'];
    auto res = " \t\t string";
    assert(stripRight(str, spaces) == res);
}


/*******************************************************************************
 * Counts the number of required elements in a range.
 * Params:
 *     range = Some input range.
 *     value = Value to find.
 * Returns: number of required elements.
 */
size_t countElements(R, E)(R range, E value) nothrow pure @safe
if (isInputRange!R)
do {
    size_t res;
    foreach(elem; range) {
        res += (value == elem);
    }
    return res;
}
unittest {
    int[] arr = [0, 1, 3, 1, 5, 8, 5, 1];
    assert(countElements(arr, 1) == 3);
    assert(countElements(arr, 5) == 2);
    assert(countElements(arr, 9) == 0);
}


/*******************************************************************************
 * Divides an array: makes a new array of arrays containing elements 
 * from source array. Each subarray contains specified number of elements.
 * Params:
 *     arr = Source array.
 *     n = Number of elements in each new subarray.
 */
T[] divideByElemNumber(T)(T arr, size_t n) nothrow pure @safe
if (std.traits.isArray!T)
in {
    assert(arr.length % n == 0);
} do {
    T[] res;
    for (size_t i = 0; i < arr.length; i += n) {
        res ~= arr[i .. i + n];
    }
    return res;
}
unittest {
    string s = "123456";
    assert(divideByElemNumber(s, 2) == ["12", "34", "56"]);
    assert(divideByElemNumber(s, 3) == ["123", "456"]);
}


/*******************************************************************************
 * Creates a new dynamic array of the same size and copies the contents of
 * the dynamic array into it. This function supports arrays with complex
 * constant structures, unlike the 'dup' function.
 * Params:
 *     arr = The dynamic array.
 * Returns:
 *     A copy of the required array.
 */
T[] copyArray(T)(const T[] arr) if (!is(T == class)) {
    T[] copy = new T[arr.length];
    for (size_t i = 0; i < arr.length; i++) {
        T elem = arr[i];
        copy[i] = elem;
    }
    return copy;
}
unittest {
    // copied for testing
    T[] copyArray(T)(const T[] arr) if (!is(T == class)) {
        T[] copy = new T[arr.length];
        for (size_t i = 0; i < arr.length; i++) {
            // it doesn't work with postblit, only with modern copy ctor
            T elem = arr[i];
            copy[i] = elem;
        }
        return copy;
    }
    struct S {
        int x, y;
        bool[] a;
        this(ref return scope const S rhs) {
            this.x = rhs.x;
            this.y = rhs.y;
            this.a = rhs.a.dup;
        }
    }
    const S[] arr = [
        S(1, 2, [true, false]),
        S(3, 4, [false, true])
    ];
    // S[] copy = arr.dup;  // It can't be compiled!
    S[] copy = copyArray(arr);   // from const array to non-const one

    assert(copy.length == arr.length);
    for (size_t i = 0; i < arr.length; i++) {
        assert(arr[i].x == copy[i].x);
        assert(arr[i].y == copy[i].y);
        assert(arr[i].a == copy[i].a);
        assert(arr[i].a.ptr != copy[i].a.ptr);
    }
}