sqlmath 2023.12.20 → 2024.3.25

package/CHANGELOG.md

@@ -4,6 +4,18 @@
 - sqlmath - Add sqlite-extension for lightgbm.
 - none
 
+# v2024.3.25
+- jslint-ci - Add shell-functions shGitPullrequestCleanup(), shGitPullrequest() to automatically cleanup or create-and-push github-pull-commit to origin/alpha.
+- ci - Fix tmpdir in shell-function shBrowserScreenshot().
+- webdemo - Update tables tradebot_intraday_xxx.
+- migration - Rename prm ydate to xdate, ytime to xtime.
+- sqlmath - Move function dbTableImportAsync from file sqlmath_browser.mjs to sqlmath.mjs.
+- ci - Update github-ci for actions/cache, actions/setup-python from nodejs v16 to nodejs v20.
+- ci - Update shell-function shRollupFetch() to fix blank date-committed.
+
+# v2024.1.21
+- sqlmath - Add function assertErrorThrownAsync().
+
 # v2023.12.20
 - chart - Improve ergonomics with import tsv.
 - betadog - bugfix - Remove over-complicated prm ttt from table y_historical.

package/README.md

@@ -2,7 +2,7 @@
 
 
 # Status
-| Branch | [master<br>(v2023.12.20)](https://github.com/sqlmath/sqlmath/tree/master) | [beta<br>(Web Demo)](https://github.com/sqlmath/sqlmath/tree/beta) | [alpha<br>(Development)](https://github.com/sqlmath/sqlmath/tree/alpha) |
+| Branch | [master<br>(v2024.3.25)](https://github.com/sqlmath/sqlmath/tree/master) | [beta<br>(Web Demo)](https://github.com/sqlmath/sqlmath/tree/beta) | [alpha<br>(Development)](https://github.com/sqlmath/sqlmath/tree/alpha) |
 |--:|:--:|:--:|:--:|
 | CI | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Amaster) | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=beta)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Abeta) | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=alpha)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Aalpha) |
 | Coverage | [![coverage](https://sqlmath.github.io/sqlmath/branch-master/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-master/.artifact/coverage/index.html) | [![coverage](https://sqlmath.github.io/sqlmath/branch-beta/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-beta/.artifact/coverage/index.html) | [![coverage](https://sqlmath.github.io/sqlmath/branch-alpha/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-alpha/.artifact/coverage/index.html) |
@@ -122,11 +122,11 @@ PORT=8080 sh jslint_ci.sh shHttpFileServer
 ```shell
 python -m build
 #
-twine upload --repository testpypi dist/sqlmath-2023.12.20*
-py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2023.12.20
+twine upload --repository testpypi dist/sqlmath-2024.3.25*
+py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2024.3.25
 #
-twine upload dist/sqlmath-2023.12.20*
-pip install sqlmath==2023.12.20
+twine upload dist/sqlmath-2024.3.25*
+pip install sqlmath==2024.3.25
 ```
 
 

package/jslint.mjs

@@ -163,7 +163,7 @@ let jslint_charset_ascii = (
     + "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"
     + "`abcdefghijklmnopqrstuvwxyz{|}~\u007f"
 );
-let jslint_edition = "v2023.11.1-beta";
+let jslint_edition = "v2024.3.1-beta";
 let jslint_export;                      // The jslint object to be exported.
 let jslint_fudge = 1;                   // Fudge starting line and starting
                                         // ... column to 1.
@@ -5487,13 +5487,15 @@ function jslint_phase3_parse(state) {
         if (
             token_nxt.id === "."
             || token_nxt.id === "?."
-            || token_nxt.id === "["
+
+// PR-459 - Allow destructuring-assignment after function-definition.
+
+            // || token_nxt.id === "["
         ) {
 
 // test_cause:
 // ["function aa(){}\n.aa", "prefix_function", "unexpected_a", ".", 1]
 // ["function aa(){}\n?.aa", "prefix_function", "unexpected_a", "?.", 1]
-// ["function aa(){}\n[]", "prefix_function", "unexpected_a", "[", 1]
 
             warn("unexpected_a");
         }
@@ -9952,6 +9954,12 @@ async function jstestDescribe(description, testFunction) {
         process.on("exit", jstestOnExit);
     }
 
+// PR-457 - Wait awhile for imports to initialize.
+
+    await new Promise(function (resolve) {
+        setTimeout(resolve);
+    });
+
 // Init jstestItList.
 
     jstestItList = [];
@@ -10005,9 +10013,7 @@ function jstestIt(description, testFunction, mode) {
         } catch (errCaught) {
             err = errCaught;
         }
-        resolve([
-            err, description, mode
-        ]);
+        resolve([err, description, mode]);
     }));
 }
 

package/package.json

@@ -38,5 +38,5 @@
     },
     "shCiArtifactUpload": 1,
     "shCiPublishNpm": 1,
-    "version": "2023.12.20"
+    "version": "2024.3.25"
 }

package/sqlmath.mjs

@@ -104,7 +104,24 @@ let {
 let sqlMessageDict = {}; // dict of web-worker-callbacks
 let sqlMessageId = 0;
 let sqlWorker;
-let version = "v2023.12.20";
+let version = "v2024.3.25";
+
+async function assertErrorThrownAsync(asyncFunc, regexp) {
+
+// This function will assert calling <asyncFunc> throws an error.
+
+    let err;
+    try {
+        await asyncFunc();
+    } catch (errCaught) {
+        err = errCaught;
+    }
+    assertOrThrow(err, "No error thrown.");
+    assertOrThrow(
+        !regexp || new RegExp(regexp).test(err.message),
+        err
+    );
+}
 
 function assertInt64(val) {
     // This function will assert <val> is within range of c99-signed-long-long.
@@ -758,7 +775,8 @@ async function dbFileLoadAsync({
 async function dbFileSaveAsync({
     db,
     dbData,
-    filename
+    filename,
+    modeNoop
 }) {
 
 // This function will save <db> to <filename>.
@@ -767,6 +785,7 @@ async function dbFileSaveAsync({
         db,
         dbData,
         filename,
+        modeNoop,
         modeSave: 1
     });
 }
@@ -829,6 +848,117 @@ async function dbOpenAsync({
     return db;
 }
 
+async function dbTableImportAsync({
+    db,
+    mode,
+    tableName,
+    textData
+}) {
+// this function will create table from imported csv/json <textData>
+    let colList;
+    let rowList;
+    let rowidList;
+    let tmp;
+    switch (mode) {
+    case "csv":
+        rowList = jsonRowListFromCsv({
+            csv: textData
+        });
+        break;
+    case "tsv":
+        rowList = [];
+        textData.trimEnd().replace((/.+/g), function (line) {
+            rowList.push(line.split("\t"));
+        });
+        break;
+    // case "json":
+    default:
+        rowList = JSON.parse(textData);
+    }
+    if (!(typeof rowList === "object" && rowList)) {
+        rowList = [];
+    }
+    // normalize rowList to list
+    if (!Array.isArray(rowList)) {
+        rowidList = [];
+        rowList = Object.entries(rowList).map(function ([
+            key, val
+        ]) {
+            rowidList.push(key);
+            return val;
+        });
+    }
+    // normalize rowList[ii] to list
+    if (rowList.length === 0) {
+        rowList.push([
+            "undefined"
+        ]);
+    }
+    if (!Array.isArray(rowList[0])) {
+        colList = Array.from(
+            new Set(
+                rowList.map(function (obj) {
+                    return Object.keys(obj);
+                }).flat()
+            )
+        );
+        rowList = rowList.map(function (obj) {
+            return colList.map(function (key) {
+                return obj[key];
+            });
+        });
+        rowList.unshift(colList);
+    }
+    // init colList
+    colList = rowList.shift();
+    // preserve rowid
+    if (rowidList) {
+        colList.unshift("rowid");
+        rowList.forEach(function (row, ii) {
+            row.unshift(rowidList[ii]);
+        });
+    }
+    // normalize colList
+    tmp = new Set();
+    colList = colList.map(function (colName) {
+        let colName2;
+        let duplicate = 0;
+        colName = "c_" + colName.toLowerCase().replace((
+            /\W/g
+        ), "_");
+        while (true) {
+            duplicate += 1;
+            colName2 = (
+                duplicate === 1
+                ? colName
+                : colName + "_" + duplicate
+            );
+            if (!tmp.has(colName2)) {
+                tmp.add(colName2);
+                return colName2;
+            }
+        }
+    });
+    // create dbtable from rowList
+    await dbExecAsync({
+        bindList: {
+            rowList: JSON.stringify(rowList)
+        },
+        db,
+        sql: (
+            rowList.length === 0
+            ? `CREATE TABLE ${tableName} (${colList.join(",")});`
+            : (
+                `CREATE TABLE ${tableName} AS SELECT `
+                + colList.map(function (colName, ii) {
+                    return "value->>" + ii + " AS " + colName;
+                }).join(",")
+                + " FROM JSON_EACH($rowList);"
+            )
+        )
+    });
+}
+
 async function fsCopyFileUnlessTest(file1, file2, mode) {
 
 // This function will copy <file1> to <file2> unless <npm_config_mode_test> = 1.
@@ -1150,6 +1280,157 @@ function jsonParseArraybuffer(buf) {
     );
 }
 
+function jsonRowListFromCsv({
+    csv
+}) {
+// this function will convert <csv>-text to json list-of-list
+//
+// https://tools.ietf.org/html/rfc4180#section-2
+// Definition of the CSV Format
+// While there are various specifications and implementations for the
+// CSV format (for ex. [4], [5], [6] and [7]), there is no formal
+// specification in existence, which allows for a wide variety of
+// interpretations of CSV files.  This section documents the format that
+// seems to be followed by most implementations:
+//
+// 1.  Each record is located on a separate line, delimited by a line
+//     break (CRLF).  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+//
+// 2.  The last record in the file may or may not have an ending line
+//     break.  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx
+//
+// 3.  There maybe an optional header line appearing as the first line
+//     of the file with the same format as normal record lines.  This
+//     header will contain names corresponding to the fields in the file
+//     and should contain the same number of fields as the records in
+//     the rest of the file (the presence or absence of the header line
+//     should be indicated via the optional "header" parameter of this
+//     MIME type).  For example:
+//     field_name,field_name,field_name CRLF
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+//
+// 4.  Within the header and each record, there may be one or more
+//     fields, separated by commas.  Each line should contain the same
+//     number of fields throughout the file.  Spaces are considered part
+//     of a field and should not be ignored.  The last field in the
+//     record must not be followed by a comma.  For example:
+//     aaa,bbb,ccc
+//
+// 5.  Each field may or may not be enclosed in double quotes (however
+//     some programs, such as Microsoft Excel, do not use double quotes
+//     at all).  If fields are not enclosed with double quotes, then
+//     double quotes may not appear inside the fields.  For example:
+//     "aaa","bbb","ccc" CRLF
+//     zzz,yyy,xxx
+//
+// 6.  Fields containing line breaks (CRLF), double quotes, and commas
+//     should be enclosed in double-quotes.  For example:
+//     "aaa","b CRLF
+//     bb","ccc" CRLF
+//     zzz,yyy,xxx
+//
+// 7.  If double-quotes are used to enclose fields, then a double-quote
+//     appearing inside a field must be escaped by preceding it with
+//     another double quote.  For example:
+//     "aaa","b""bb","ccc"
+    let match;
+    let quote;
+    let rgx;
+    let row;
+    let rowList;
+    let val;
+    // normalize "\r\n" to "\n"
+    csv = csv.trimEnd().replace((
+        /\r\n?/gu
+    ), "\n") + "\n";
+    rgx = (
+        /(.*?)(""|"|,|\n)/gu
+    );
+    rowList = [];
+    // reset row
+    row = [];
+    val = "";
+    while (true) {
+        match = rgx.exec(csv);
+        if (!match) {
+// 2.  The last record in the file may or may not have an ending line
+//     break.  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx
+            if (!row.length) {
+                break;
+            }
+            // // if eof missing crlf, then mock it
+            // rgx.lastIndex = csv.length;
+            // match = [
+            //     "\n", "", "\n"
+            // ];
+        }
+        // build val
+        val += match[1];
+        if (match[2] === "\"") {
+// 5.  Each field may or may not be enclosed in double quotes (however
+//     some programs, such as Microsoft Excel, do not use double quotes
+//     at all).  If fields are not enclosed with double quotes, then
+//     double quotes may not appear inside the fields.  For example:
+//     "aaa","bbb","ccc" CRLF
+//     zzz,yyy,xxx
+            quote = !quote;
+        } else if (quote) {
+// 7.  If double-quotes are used to enclose fields, then a double-quote
+//     appearing inside a field must be escaped by preceding it with
+//     another double quote.  For example:
+//     "aaa","b""bb","ccc"
+            if (match[2] === "\"\"") {
+                val += "\"";
+// 6.  Fields containing line breaks (CRLF), double quotes, and commas
+//     should be enclosed in double-quotes.  For example:
+//     "aaa","b CRLF
+//     bb","ccc" CRLF
+//     zzz,yyy,xxx
+            } else {
+                val += match[2];
+            }
+        } else if (match[2] === ",") {
+// 4.  Within the header and each record, there may be one or more
+//     fields, separated by commas.  Each line should contain the same
+//     number of fields throughout the file.  Spaces are considered part
+//     of a field and should not be ignored.  The last field in the
+//     record must not be followed by a comma.  For example:
+//     aaa,bbb,ccc
+            // delimit val
+            row.push(val);
+            val = "";
+        } else if (match[2] === "\n") {
+// 1.  Each record is located on a separate line, delimited by a line
+//     break (CRLF).  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+            // delimit val
+            row.push(val);
+            val = "";
+            // append row
+            rowList.push(row);
+            // reset row
+            row = [];
+        }
+    }
+    // // append val
+    // if (val) {
+    //     row.push(val);
+    // }
+    // // append row
+    // if (row.length) {
+    //     rowList.push(row);
+    // }
+    return rowList;
+}
+
 async function moduleFsInit() {
 
 // This function will import nodejs builtin-modules if they have not yet been
@@ -1368,6 +1649,7 @@ export {
     SQLITE_OPEN_TRANSIENT_DB,
     SQLITE_OPEN_URI,
     SQLITE_OPEN_WAL,
+    assertErrorThrownAsync,
     assertInt64,
     assertJsonEqual,
     assertNumericalEqual,
@@ -1381,6 +1663,7 @@ export {
     dbFileSaveAsync,
     dbNoopAsync,
     dbOpenAsync,
+    dbTableImportAsync,
     debugInline,
     fsCopyFileUnlessTest,
     fsExistsUnlessTest,