sqlmath 2024.1.21 → 2024.5.26

package/CHANGELOG.md

@@ -4,6 +4,26 @@
 - sqlmath - Add sqlite-extension for lightgbm.
 - none
 
+# v2024.5.26
+- webdemo - Replace market-indices .spx/.ndx/.dji with futures .es/.nq/.ym.
+- sqlite - Decouple c-function str99JsonAppendText() from builtin-function jsonAppendString().
+- sqlite - Update to sqlite v3.42.0.
+- sqlmath - Remove little-used sqlean-regexp-extension and file sqlmath_external_pcre2.c.
+- sqlmath - bugfix - Fix off-by-2 root-mean-square-deviation calculation of parameter lee in sql-function win_sinefit2().
+- sqlmath - bugfix - Fix null-case handling-behavior for function dbExecAndReturnLastBlobAsync().
+- sqlmath - Add functions dbExecAndReturnFirstRow(), dbExecAndReturnFirstTable(), listOrEmptyList().
+- jslint - Update jslint to v2024.4.1-beta.
+- ci - bugfix - Fix package.json config-regression breaking ci.
+
+# v2024.3.25
+- jslint-ci - Add shell-functions shGitPullrequestCleanup(), shGitPullrequest() to automatically cleanup or create-and-push github-pull-commit to origin/alpha.
+- ci - Fix tmpdir in shell-function shBrowserScreenshot().
+- webdemo - Update tables tradebot_intraday_xxx.
+- migration - Rename prm ydate to xdate, ytime to xtime.
+- sqlmath - Move function dbTableImportAsync from file sqlmath_browser.mjs to sqlmath.mjs.
+- ci - Update github-ci for actions/cache, actions/setup-python from nodejs v16 to nodejs v20.
+- ci - Update shell-function shRollupFetch() to fix blank date-committed.
+
 # v2024.1.21
 - sqlmath - Add function assertErrorThrownAsync().
 
@@ -24,7 +44,7 @@
 - sqlean - Add sqlean-extension regexp with regexp-replacement and pcre2 - increases wasm size to 1.1mb.
 - zlib - Update to zlib v1.3.
 - sqlmath - Fix SIGSEGV error when binding external-buffer during db_exec.
-- sqlite - Update to sqlite v3.39.4.
+- sqlite - Update to sqlite v3.42.0.
 - python - Add python-functions db_file_load(), db_file_save().
 - sqlmath - Revamp how js-arraybuffers are passed to c-api without copying.
 - python - Revamp python-c-extension to support cp312.

package/README.md

@@ -2,7 +2,7 @@
 
 
 # Status
-| Branch | [master<br>(v2024.1.21)](https://github.com/sqlmath/sqlmath/tree/master) | [beta<br>(Web Demo)](https://github.com/sqlmath/sqlmath/tree/beta) | [alpha<br>(Development)](https://github.com/sqlmath/sqlmath/tree/alpha) |
+| Branch | [master<br>(v2024.5.26)](https://github.com/sqlmath/sqlmath/tree/master) | [beta<br>(Web Demo)](https://github.com/sqlmath/sqlmath/tree/beta) | [alpha<br>(Development)](https://github.com/sqlmath/sqlmath/tree/alpha) |
 |--:|:--:|:--:|:--:|
 | CI | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=master)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Amaster) | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=beta)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Abeta) | [![ci](https://github.com/sqlmath/sqlmath/actions/workflows/ci.yml/badge.svg?branch=alpha)](https://github.com/sqlmath/sqlmath/actions?query=branch%3Aalpha) |
 | Coverage | [![coverage](https://sqlmath.github.io/sqlmath/branch-master/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-master/.artifact/coverage/index.html) | [![coverage](https://sqlmath.github.io/sqlmath/branch-beta/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-beta/.artifact/coverage/index.html) | [![coverage](https://sqlmath.github.io/sqlmath/branch-alpha/.artifact/coverage/coverage_badge.svg)](https://sqlmath.github.io/sqlmath/branch-alpha/.artifact/coverage/index.html) |
@@ -106,7 +106,6 @@ PORT=8080 sh jslint_ci.sh shHttpFileServer
 # License
 - [sqlite](https://github.com/sqlite/sqlite) is under [public domain](https://www.sqlite.org/copyright.html).
 - [jslint](https://github.com/jslint-org/jslint) is under [Unlicense License](https://github.com/jslint-org/jslint/blob/master/LICENSE).
-- [pcre2](https://github.com/PCRE2Project/pcre2) is under [3-Clause BSD License](https://github.com/PCRE2Project/pcre2/blob/pcre2-10.42/LICENCE)
 - [zlib](https://github.com/madler/zlib) is under [zlib License](https://github.com/madler/zlib/blob/v1.2.13/LICENSE).
 - [cpplint.py](cpplint.py) is under [3-Clause BSD License](https://github.com/cpplint/cpplint/blob/1.5.5/LICENSE).
 - [indent.exe](indent.exe) is under [GPLv3 License](https://www.gnu.org/licenses/gpl-3.0.txt).
@@ -122,11 +121,11 @@ PORT=8080 sh jslint_ci.sh shHttpFileServer
 ```shell
 python -m build
 #
-twine upload --repository testpypi dist/sqlmath-2024.1.21*
-py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2024.1.21
+twine upload --repository testpypi dist/sqlmath-2024.5.26*
+py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2024.5.26
 #
-twine upload dist/sqlmath-2024.1.21*
-pip install sqlmath==2024.1.21
+twine upload dist/sqlmath-2024.5.26*
+pip install sqlmath==2024.5.26
 ```
 
 
@@ -134,13 +133,16 @@ pip install sqlmath==2024.1.21
 ### sqlite upgrade
 - goto https://www.sqlite.org/changes.html
 ```shell
-curl -L https://www.sqlite.org/2023/sqlite-autoconf-3420000.tar.gz | tar -xz
-git grep "3\.39\.4\|3390400"
-for FILE in .ci.sh sqlite_rollup.c
+curl -L https://www.sqlite.org/2023/sqlite-autoconf-3440200.tar.gz | tar -xz
+mv sqlite-autoconf-3440200 .sqlite-autoconf-3440200
+git grep "3\.42\.0\|3420000"
+for FILE in .ci.sh sqlmath_external_sqlite.c sqlmath_external_zlib.c
 do
-    sed -i -e "s|\<3\.39\.4\>|3.42.0|g" "$FILE"
-    sed -i -e "s|\<3390400\>|3420000|g" "$FILE"
+    sed -i -e "s|\<3\.42\.0\>|3.44.2|g" "$FILE"
+    sed -i -e "s|\<3420000\>|3440200|g" "$FILE"
 done
-git grep "3\.39\.4\|3390400"
-shRollupFetch sqlite_rollup.c
+git grep "3\.42\.0\|3420000"
+shRollupFetch sqlmath_external_sqlite.c
+shRollupFetch sqlmath_external_zlib.c
+sh jslint_ci.sh shSqlmathUpdate
 ```

package/jslint.mjs

@@ -163,7 +163,7 @@ let jslint_charset_ascii = (
     + "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"
     + "`abcdefghijklmnopqrstuvwxyz{|}~\u007f"
 );
-let jslint_edition = "v2023.11.1-beta";
+let jslint_edition = "v2024.4.1-beta";
 let jslint_export;                      // The jslint object to be exported.
 let jslint_fudge = 1;                   // Fudge starting line and starting
                                         // ... column to 1.
@@ -5487,13 +5487,15 @@ function jslint_phase3_parse(state) {
         if (
             token_nxt.id === "."
             || token_nxt.id === "?."
-            || token_nxt.id === "["
+
+// PR-459 - Allow destructuring-assignment after function-definition.
+
+            // || token_nxt.id === "["
         ) {
 
 // test_cause:
 // ["function aa(){}\n.aa", "prefix_function", "unexpected_a", ".", 1]
 // ["function aa(){}\n?.aa", "prefix_function", "unexpected_a", "?.", 1]
-// ["function aa(){}\n[]", "prefix_function", "unexpected_a", "[", 1]
 
             warn("unexpected_a");
         }
@@ -9952,6 +9954,12 @@ async function jstestDescribe(description, testFunction) {
         process.on("exit", jstestOnExit);
     }
 
+// PR-457 - Wait awhile for imports to initialize.
+
+    await new Promise(function (resolve) {
+        setTimeout(resolve);
+    });
+
 // Init jstestItList.
 
     jstestItList = [];
@@ -10005,9 +10013,7 @@ function jstestIt(description, testFunction, mode) {
         } catch (errCaught) {
             err = errCaught;
         }
-        resolve([
-            err, description, mode
-        ]);
+        resolve([err, description, mode]);
     }));
 }
 

package/package.json

@@ -11,7 +11,7 @@
     "engines": {
         "node": ">=14"
     },
-    "fileCount": 42,
+    "fileCount": 41,
     "homepage": "https://github.com/sqlmath/sqlmath",
     "keywords": [
         "data-science",
@@ -29,7 +29,7 @@
     ],
     "repository": {
         "type": "git",
-        "url": "https://github.com/sqlmath/sqlmath.git"
+        "url": "git+https://github.com/sqlmath/sqlmath.git"
     },
     "scripts": {
         "build": "sh jslint_ci.sh shCiBuildNodejs",
@@ -38,5 +38,5 @@
     },
     "shCiArtifactUpload": 1,
     "shCiPublishNpm": 1,
-    "version": "2024.1.21"
+    "version": "2024.5.26"
 }

package/sqlmath.mjs

@@ -92,6 +92,7 @@ let debugInline = (function () {
 }());
 let moduleChildProcess;
 let moduleChildProcessSpawn;
+let moduleCrypto;
 let moduleFs;
 let moduleFsInitResolveList;
 let modulePath;
@@ -104,7 +105,7 @@ let {
 let sqlMessageDict = {}; // dict of web-worker-callbacks
 let sqlMessageId = 0;
 let sqlWorker;
-let version = "v2024.1.21";
+let version = "v2024.5.26";
 
 async function assertErrorThrownAsync(asyncFunc, regexp) {
 
@@ -341,7 +342,6 @@ async function ciBuildExt1NodejsConfigure({
                 ],
                 "sources": [
                     "sqlmath_base.c",
-                    "sqlmath_external_pcre2.c",
                     "sqlmath_external_sqlite.c",
                     "sqlmath_external_zlib.c"
                 ],
@@ -626,14 +626,59 @@ async function dbCloseAsync(db) {
     }));
 }
 
+async function dbExecAndReturnFirstRow({
+    bindList = [],
+    db,
+    sql
+}) {
+
+// This function will exec <sql> in <db>,
+// and return first-row or empty-object.
+
+    return (
+        noop(
+            noop(
+                await dbExecAsync({
+                    bindList,
+                    db,
+                    sql
+                })
+            )[0]
+            || []
+        )[0]
+        || {}
+    );
+}
+
+async function dbExecAndReturnFirstTable({
+    bindList = [],
+    db,
+    sql
+}) {
+
+// This function will exec <sql> in <db>,
+// and return first-table or empty-list.
+
+    return (
+        noop(
+            await dbExecAsync({
+                bindList,
+                db,
+                sql
+            })
+        )[0]
+        || []
+    );
+}
+
 function dbExecAndReturnLastBlobAsync({
     bindList = [],
     db,
     sql
 }) {
 
-// This function will exec <sql> in <db> and return last value retrieved
-// from execution as raw blob/buffer.
+// This function will exec <sql> in <db>,
+// and return last-value retrieved from execution as raw blob/buffer.
 
     return dbExecAsync({
         bindList,
@@ -743,6 +788,25 @@ async function dbFileLoadAsync({
 
 // This function will load <filename> to <db>.
 
+    let filename2;
+    async function _dbFileLoad() {
+        dbData = await dbCallAsync(
+            jsbatonCreate("_dbFileLoad"),
+            [
+                // 0. sqlite3 * pInMemory
+                db,
+                // 1. char *zFilename
+                filename,
+                // 2. const int isSave
+                modeSave,
+                // 3. undefined
+                undefined,
+                // 4. dbData - same position as dbOpenAsync
+                dbData
+            ],
+            "modeDb"
+        );
+    }
     if (modeNoop) {
         return;
     }
@@ -753,29 +817,30 @@ async function dbFileLoadAsync({
         typeof filename === "string" && filename,
         `invalid filename ${filename}`
     );
-    dbData = await dbCallAsync(
-        jsbatonCreate("_dbFileLoad"),
-        [
-            // 0. sqlite3 * pInMemory
-            db,
-            // 1. char *zFilename
-            filename,
-            // 2. const int isSave
-            modeSave,
-            // 3. undefined
-            undefined,
-            // 4. dbData - same position as dbOpenAsync
-            dbData
-        ],
-        "modeDb"
-    );
+    // Save to tmpfile and then atomically-rename to actual-filename.
+    if (moduleFs && modeSave) {
+        filename2 = filename;
+        filename = modulePath.join(
+            modulePath.dirname(filename),
+            `.dbFileSaveAsync.${moduleCrypto.randomUUID()}`
+        );
+        try {
+            await _dbFileLoad();
+            await moduleFs.promises.rename(filename, filename2);
+        } finally {
+            await moduleFs.promises.unlink(filename).catch(noop);
+        }
+    } else {
+        await _dbFileLoad();
+    }
     return dbData[1 + 0];
 }
 
 async function dbFileSaveAsync({
     db,
     dbData,
-    filename
+    filename,
+    modeNoop
 }) {
 
 // This function will save <db> to <filename>.
@@ -784,6 +849,7 @@ async function dbFileSaveAsync({
         db,
         dbData,
         filename,
+        modeNoop,
         modeSave: 1
     });
 }
@@ -846,6 +912,117 @@ async function dbOpenAsync({
     return db;
 }
 
+async function dbTableImportAsync({
+    db,
+    mode,
+    tableName,
+    textData
+}) {
+// this function will create table from imported csv/json <textData>
+    let colList;
+    let rowList;
+    let rowidList;
+    let tmp;
+    switch (mode) {
+    case "csv":
+        rowList = jsonRowListFromCsv({
+            csv: textData
+        });
+        break;
+    case "tsv":
+        rowList = [];
+        textData.trimEnd().replace((/.+/g), function (line) {
+            rowList.push(line.split("\t"));
+        });
+        break;
+    // case "json":
+    default:
+        rowList = JSON.parse(textData);
+    }
+    if (!(typeof rowList === "object" && rowList)) {
+        rowList = [];
+    }
+    // normalize rowList to list
+    if (!Array.isArray(rowList)) {
+        rowidList = [];
+        rowList = Object.entries(rowList).map(function ([
+            key, val
+        ]) {
+            rowidList.push(key);
+            return val;
+        });
+    }
+    // normalize rowList[ii] to list
+    if (rowList.length === 0) {
+        rowList.push([
+            "undefined"
+        ]);
+    }
+    if (!Array.isArray(rowList[0])) {
+        colList = Array.from(
+            new Set(
+                rowList.map(function (obj) {
+                    return Object.keys(obj);
+                }).flat()
+            )
+        );
+        rowList = rowList.map(function (obj) {
+            return colList.map(function (key) {
+                return obj[key];
+            });
+        });
+        rowList.unshift(colList);
+    }
+    // init colList
+    colList = rowList.shift();
+    // preserve rowid
+    if (rowidList) {
+        colList.unshift("rowid");
+        rowList.forEach(function (row, ii) {
+            row.unshift(rowidList[ii]);
+        });
+    }
+    // normalize colList
+    tmp = new Set();
+    colList = colList.map(function (colName) {
+        let colName2;
+        let duplicate = 0;
+        colName = "c_" + colName.toLowerCase().replace((
+            /\W/g
+        ), "_");
+        while (true) {
+            duplicate += 1;
+            colName2 = (
+                duplicate === 1
+                ? colName
+                : colName + "_" + duplicate
+            );
+            if (!tmp.has(colName2)) {
+                tmp.add(colName2);
+                return colName2;
+            }
+        }
+    });
+    // create dbtable from rowList
+    await dbExecAsync({
+        bindList: {
+            rowList: JSON.stringify(rowList)
+        },
+        db,
+        sql: (
+            rowList.length === 0
+            ? `CREATE TABLE ${tableName} (${colList.join(",")});`
+            : (
+                `CREATE TABLE ${tableName} AS SELECT `
+                + colList.map(function (colName, ii) {
+                    return "value->>" + ii + " AS " + colName;
+                }).join(",")
+                + " FROM JSON_EACH($rowList);"
+            )
+        )
+    });
+}
+
 async function fsCopyFileUnlessTest(file1, file2, mode) {
 
 // This function will copy <file1> to <file2> unless <npm_config_mode_test> = 1.
@@ -1167,6 +1344,164 @@ function jsonParseArraybuffer(buf) {
     );
 }
 
+function jsonRowListFromCsv({
+    csv
+}) {
+// this function will convert <csv>-text to json list-of-list
+//
+// https://tools.ietf.org/html/rfc4180#section-2
+// Definition of the CSV Format
+// While there are various specifications and implementations for the
+// CSV format (for ex. [4], [5], [6] and [7]), there is no formal
+// specification in existence, which allows for a wide variety of
+// interpretations of CSV files.  This section documents the format that
+// seems to be followed by most implementations:
+//
+// 1.  Each record is located on a separate line, delimited by a line
+//     break (CRLF).  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+//
+// 2.  The last record in the file may or may not have an ending line
+//     break.  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx
+//
+// 3.  There maybe an optional header line appearing as the first line
+//     of the file with the same format as normal record lines.  This
+//     header will contain names corresponding to the fields in the file
+//     and should contain the same number of fields as the records in
+//     the rest of the file (the presence or absence of the header line
+//     should be indicated via the optional "header" parameter of this
+//     MIME type).  For example:
+//     field_name,field_name,field_name CRLF
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+//
+// 4.  Within the header and each record, there may be one or more
+//     fields, separated by commas.  Each line should contain the same
+//     number of fields throughout the file.  Spaces are considered part
+//     of a field and should not be ignored.  The last field in the
+//     record must not be followed by a comma.  For example:
+//     aaa,bbb,ccc
+//
+// 5.  Each field may or may not be enclosed in double quotes (however
+//     some programs, such as Microsoft Excel, do not use double quotes
+//     at all).  If fields are not enclosed with double quotes, then
+//     double quotes may not appear inside the fields.  For example:
+//     "aaa","bbb","ccc" CRLF
+//     zzz,yyy,xxx
+//
+// 6.  Fields containing line breaks (CRLF), double quotes, and commas
+//     should be enclosed in double-quotes.  For example:
+//     "aaa","b CRLF
+//     bb","ccc" CRLF
+//     zzz,yyy,xxx
+//
+// 7.  If double-quotes are used to enclose fields, then a double-quote
+//     appearing inside a field must be escaped by preceding it with
+//     another double quote.  For example:
+//     "aaa","b""bb","ccc"
+    let match;
+    let quote;
+    let rgx;
+    let row;
+    let rowList;
+    let val;
+    // normalize "\r\n" to "\n"
+    csv = csv.trimEnd().replace((
+        /\r\n?/gu
+    ), "\n") + "\n";
+    rgx = (
+        /(.*?)(""|"|,|\n)/gu
+    );
+    rowList = [];
+    // reset row
+    row = [];
+    val = "";
+    while (true) {
+        match = rgx.exec(csv);
+        if (!match) {
+// 2.  The last record in the file may or may not have an ending line
+//     break.  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx
+            if (!row.length) {
+                break;
+            }
+            // // if eof missing crlf, then mock it
+            // rgx.lastIndex = csv.length;
+            // match = [
+            //     "\n", "", "\n"
+            // ];
+        }
+        // build val
+        val += match[1];
+        if (match[2] === "\"") {
+// 5.  Each field may or may not be enclosed in double quotes (however
+//     some programs, such as Microsoft Excel, do not use double quotes
+//     at all).  If fields are not enclosed with double quotes, then
+//     double quotes may not appear inside the fields.  For example:
+//     "aaa","bbb","ccc" CRLF
+//     zzz,yyy,xxx
+            quote = !quote;
+        } else if (quote) {
+// 7.  If double-quotes are used to enclose fields, then a double-quote
+//     appearing inside a field must be escaped by preceding it with
+//     another double quote.  For example:
+//     "aaa","b""bb","ccc"
+            if (match[2] === "\"\"") {
+                val += "\"";
+// 6.  Fields containing line breaks (CRLF), double quotes, and commas
+//     should be enclosed in double-quotes.  For example:
+//     "aaa","b CRLF
+//     bb","ccc" CRLF
+//     zzz,yyy,xxx
+            } else {
+                val += match[2];
+            }
+        } else if (match[2] === ",") {
+// 4.  Within the header and each record, there may be one or more
+//     fields, separated by commas.  Each line should contain the same
+//     number of fields throughout the file.  Spaces are considered part
+//     of a field and should not be ignored.  The last field in the
+//     record must not be followed by a comma.  For example:
+//     aaa,bbb,ccc
+            // delimit val
+            row.push(val);
+            val = "";
+        } else if (match[2] === "\n") {
+// 1.  Each record is located on a separate line, delimited by a line
+//     break (CRLF).  For example:
+//     aaa,bbb,ccc CRLF
+//     zzz,yyy,xxx CRLF
+            // delimit val
+            row.push(val);
+            val = "";
+            // append row
+            rowList.push(row);
+            // reset row
+            row = [];
+        }
+    }
+    // // append val
+    // if (val) {
+    //     row.push(val);
+    // }
+    // // append row
+    // if (row.length) {
+    //     rowList.push(row);
+    // }
+    return rowList;
+}
+
+function listOrEmptyList(list) {
+
+// This function will return <list> or empty-list if falsy.
+
+    return list || [];
+}
+
 async function moduleFsInit() {
 
 // This function will import nodejs builtin-modules if they have not yet been
@@ -1191,11 +1526,13 @@ async function moduleFsInit() {
     moduleFsInitResolveList = [];
     [
         moduleChildProcess,
+        moduleCrypto,
         moduleFs,
         modulePath,
         moduleUrl
     ] = await Promise.all([
         import("child_process"),
+        import("crypto"),
         import("fs"),
         import("path"),
         import("url")
@@ -1359,6 +1696,15 @@ function sqlmathWebworkerInit({
     }
 }
 
+function waitAsync(timeout) {
+
+// This function will wait <timeout> ms.
+
+    return new Promise(function (resolve) {
+        setTimeout(resolve, timeout * !npm_config_mode_test);
+    });
+}
+
 sqlmathInit(); // coverage-hack
 await sqlmathInit();
 sqlmathInit(); // coverage-hack
@@ -1393,12 +1739,15 @@ export {
     childProcessSpawn2,
     ciBuildExt,
     dbCloseAsync,
+    dbExecAndReturnFirstRow,
+    dbExecAndReturnFirstTable,
     dbExecAndReturnLastBlobAsync,
     dbExecAsync,
     dbFileLoadAsync,
     dbFileSaveAsync,
     dbNoopAsync,
     dbOpenAsync,
+    dbTableImportAsync,
     debugInline,
     fsCopyFileUnlessTest,
     fsExistsUnlessTest,
@@ -1406,8 +1755,10 @@ export {
     fsWriteFileUnlessTest,
     jsbatonGetInt64,
     jsbatonGetString,
+    listOrEmptyList,
     noop,
     objectDeepCopyWithKeysSorted,
     sqlmathWebworkerInit,
-    version
+    version,
+    waitAsync
 };