sqlmath 0.0.5 → 2022.4.28

package/package.json

@@ -3,7 +3,7 @@
         "email": "kaizhu256@gmail.com",
         "name": "kai zhu"
     },
-    "counter": 1,
+    "counter": 10,
     "cpu": [
         "x64"
     ],
@@ -11,7 +11,7 @@
     "engines": {
         "node": ">=14"
     },
-    "fileCount": 22,
+    "fileCount": 28,
     "homepage": "https://github.com/sqlmath/sqlmath",
     "keywords": [
         "data-science",
@@ -32,10 +32,10 @@
         "url": "https://github.com/sqlmath/sqlmath"
     },
     "scripts": {
-        "build": "sh jslint_ci.sh shCiBuild",
-        "test": "sh jslint_ci.sh shCiTest",
+        "build": "sh jslint_ci.sh shCiBuildNodejs",
+        "test": "sh jslint_ci.sh shCiTestNodejs",
         "test2": "sh jslint_ci.sh shCiBase",
         "test_win32": "node -e \"require('child_process').spawn('C:\\\\Program Files\\\\Git\\\\bin\\\\bash.exe',['-c','npm run test' + ' ' + process.argv.slice(1).join(' ')],{stdio:['ignore',1,2]});\""
     },
-    "version": "0.0.5"
+    "version": "2022.4.28"
 }

package/sqlmath.mjs

@@ -8,7 +8,8 @@ let {
     assertJsonEqual,
     assertOrThrow,
     debugInline,
-    noop
+    noop,
+    objectDeepCopyWithKeysSorted
 } = jslint;
 let local = Object.assign({}, jslint);
 
@@ -17,7 +18,7 @@ function assertNumericalEqual(aa, bb, message) {
 // This function will assert aa - bb <= Number.EPSILON
 
     assertOrThrow(aa, "value cannot be 0 or falsy");
-    if (!(Math.abs(aa - bb) <= Number.EPSILON)) {
+    if (!(Math.abs((aa - bb) / Math.max(aa, bb)) <= 256 * Number.EPSILON)) {
         throw new Error(
             JSON.stringify(aa) + " != " + JSON.stringify(bb) + (
                 message
@@ -58,10 +59,10 @@ file sqlmath.js
     let SQLITE_OPEN_URI = 0x00000040;           /* Ok for sqlite3_open_v2() */
     let SQLITE_OPEN_WAL = 0x00080000;           /* VFS only */
     let addon;
-    let dbDict = new WeakMap();
+    let consoleError = console.error;
+    let dbDict = new WeakMap(); // private map of sqlite-database-connections
     let requireCjs = createRequire(import.meta.url);
     let testList;
-    // private map of sqlite-database-connections
 
     function cCall(func, argList) {
 // this function will serialize <argList> to a c <baton>,
@@ -168,8 +169,12 @@ file sqlmath.js
         bindList = [],
         db,
         responseType,
-        rowList,
-        sql
+        sql,
+        tmpColList,
+        tmpColListPriority,
+        tmpCsv,
+        tmpRowList,
+        tmpTableName
     }) {
 // this function will exec <sql> in <db> and return <result>
         let bindByKey = !Array.isArray(bindList);
@@ -180,10 +185,14 @@ file sqlmath.js
         );
         let result;
         let serialize = jsToSqlSerializer();
-        if (rowList) {
+        if (tmpCsv || tmpRowList) {
             await dbTableInsertAsync({
+                colList: tmpColList,
+                colListPriority: tmpColListPriority,
+                csv: tmpCsv,
                 db,
-                rowList
+                rowList: tmpRowList,
+                tableName: tmpTableName
             });
         }
         Object.entries(bindList).forEach(function ([
@@ -202,8 +211,6 @@ file sqlmath.js
             (
                 responseType === "lastBlob"
                 ? 1
-                : responseType === "lastMatrixDouble"
-                ? 2
                 : 0
             )
         ].concat(serialize.bufSharedList));
@@ -212,8 +219,6 @@ file sqlmath.js
         case "arraybuffer":
         case "lastBlob":
             return result;
-        case "lastMatrixDouble":
-            return new Float64Array(result);
         case "list":
             return JSON.parse(new TextDecoder().decode(result));
         default:
@@ -231,6 +236,25 @@ file sqlmath.js
         }
     }
 
+    async function dbExecWithRetryAsync(option) {
+// this function will exec <sql> in <db> and return <result> with <retryLimit>
+        let retry = option.retryLimit || 1;
+        while (true) {
+            try {
+                return await dbExecAsync(option);
+            } catch (err) {
+                assertOrThrow(retry > 0, err);
+                consoleError(err);
+                consoleError(
+                    "dbExecWithRetryAsync - retry failed sql-query with "
+                    + retry
+                    + " remaining retry"
+                );
+                retry -= 1;
+            }
+        }
+    }
+
     function dbGetLastBlobAsync({
         bindList = [],
         db,
@@ -246,21 +270,6 @@ file sqlmath.js
         });
     }
 
-    function dbGetLastMatrixDouble({
-        bindList = [],
-        db,
-        sql
-    }) {
-// this function will exec <sql> in <db> and return last SELECT-statement
-// from execution as row x col matrix of doubles
-        return dbExecAsync({
-            bindList,
-            db,
-            responseType: "lastMatrixDouble",
-            sql
-        });
-    }
-
     async function dbMemoryLoadAsync({
         db,
         filename
@@ -306,9 +315,12 @@ file sqlmath.js
         ), async function () {
             let finalizer;
             let ptr = await cCall("__dbOpenAsync", [
-                String(filename), undefined, flags ?? (
+                String(filename),
+                undefined,
+                flags ?? (
                     SQLITE_OPEN_CREATE | SQLITE_OPEN_READWRITE | SQLITE_OPEN_URI
-                ), undefined
+                ),
+                undefined
             ]);
             ptr = ptr[0][0];
             finalizer = new BigInt64Array(addon.__dbFinalizerCreate());
@@ -330,13 +342,14 @@ file sqlmath.js
         csv,
         db,
         rowList,
-        tableName = "tmp1"
+        tableName
     }) {
 // this function will create-or-replace temp <tablename> with <rowList>
         let serialize = jsToSqlSerializer();
         let sqlCreateTable;
         let sqlInsertRow;
         // normalize and validate tableName
+        tableName = tableName || "__tmp1";
         tableName = "temp." + JSON.stringify(tableName.replace((
             /^temp\./
         ), ""));
@@ -429,9 +442,6 @@ file sqlmath.js
         }
         function bufferSetBuffer(aa, bb, offset) {
 // this function will set buffer <bb> to buffer <aa> at <offset>
-            if (typeof bb === "string") {
-                bb = new TextEncoder().encode(bb);
-            }
             aa = new Uint8Array(aa.buffer, aa.byteOffset, aa.byteLength);
             bb = new Uint8Array(bb.buffer, bb.byteOffset, bb.byteLength);
             aa.set(bb, offset);
@@ -1013,50 +1023,6 @@ Definition of the CSV Format
                     dd
                 });
             });
-            // test dbGetLastMatrixDouble's bind handling-behavior
-            [
-                aa
-            ].forEach(async function (aa) {
-                let cc = Number(
-                    typeof aa === "symbol"
-                    ? 0
-                    : aa
-                ) || 0;
-                let dd = await dbGetLastMatrixDouble({
-                    bindList: [
-                        aa
-                    ],
-                    db,
-                    sql: "SELECT 1, 2, 3; SELECT 1, 2, ?"
-                });
-                switch (typeof(aa)) {
-                case "bigint":
-                    aa = String(aa);
-                    break;
-                case "object":
-                    if (typeof aa?.getUTCFullYear === "function") {
-                        cc = aa.getUTCFullYear();
-                    }
-                    break;
-                }
-                cc = new Float64Array([
-                    1, 3, 1, 2, cc
-                ]);
-                // debugInline(ii, aa, bb, cc, dd);
-                cc.forEach(function (val, jj) {
-                    assertJsonEqual(
-                        val,
-                        dd[jj],
-                        {
-                            ii,
-                            aa, //jslint-quiet
-                            bb,
-                            cc,
-                            dd
-                        }
-                    );
-                });
-            });
             // test dbExecAsync's responseType handling-behavior
             [
                 "arraybuffer",
@@ -1212,19 +1178,15 @@ Definition of the CSV Format
                 colListPriority,
                 rowList
             }, jj) {
-                let cc;
-                await dbTableInsertAsync({
-                    colList,
-                    colListPriority,
-                    db,
-                    rowList,
-                    tableName: "datatype_" + ii + "_" + jj
-                });
-                cc = noop(
+                let cc = noop(
                     await dbExecAsync({
                         db,
                         responseType: "list",
-                        sql: "SELECT * FROM datatype_" + ii + "_" + jj
+                        sql: `SELECT * FROM datatype_${ii}_${jj}`,
+                        tmpColList: colList,
+                        tmpColListPriority: colListPriority,
+                        tmpRowList: rowList,
+                        tmpTableName: "datatype_" + ii + "_" + jj
                     })
                 )[0];
                 // debugInline(ii, jj, aa, bb, cc);
@@ -1282,12 +1244,14 @@ Definition of the CSV Format
                         sql: (`
 CREATE TABLE testDbExecAsync1 AS
 SELECT 101 AS c101, 102 AS c102
+--
 UNION ALL
 VALUES
     (201, 202),
     (301, NULL);
 CREATE TABLE testDbExecAsync2 AS
 SELECT 401 AS c401, 402 AS c402, 403 AS c403
+--
 UNION ALL
 VALUES
     (501, 502.0123, 5030123456789),
@@ -1350,18 +1314,27 @@ SELECT * FROM testDbExecAsync2;
         ));
         // test sqlmath-defined-func handling-behavior
         [
+            "COT(NULL)", null,
             "COT('-1')", -0.642092615934331,
             "COT('0')", null,
             "COT('1')", 0.642092615934331,
             "COT(-1)", -0.642092615934331,
             "COT(0)", null,
             "COT(1)", 0.642092615934331,
+            "COTH(NULL)", null,
             "COTH('-1')", -1.31303528549933,
             "COTH('0')", null,
             "COTH('1')", 1.31303528549933,
             "COTH(-1)", -1.31303528549933,
             "COTH(0)", null,
             "COTH(1)", 1.31303528549933,
+            "ROUNDORZERO(NULL, NULL)", 0,
+            "ROUNDORZERO(NULL, 0)", 0,
+            "ROUNDORZERO(NULL, 0.5)", 0,
+            "ROUNDORZERO(0.5, NULL)", 1,
+            "ROUNDORZERO(0.5, 0.5)", 1,
+            "ROUNDORZERO(0.5, 1)", 0.5,
+            "SIGN(NULL)", null,
             "SIGN('-1')", -1,
             "SIGN('0')", 0,
             "SIGN('1')", 1,
@@ -1374,7 +1347,6 @@ SELECT * FROM testDbExecAsync2;
             "SIGN(0xffffffffffffffff)", -1,
             "SIGN(1)", 1,
             "SIGN(1e999)", 1,
-            "SIGN(NULL)", null,
             // sentinel
             "NULL", null
         ].forEach(async function (sql, ii, list) {
@@ -1388,15 +1360,26 @@ SELECT * FROM testDbExecAsync2;
                 await dbExecAsync({
                     db,
                     responseType: "dict",
-                    sql: "SELECT " + sql + " AS val"
+                    sql: `SELECT ${sql} AS val`
                 })
             )[0][0].val;
-            assertOrThrow(bb === cc, JSON.stringify([
-                ii, sql, bb, cc
-            ]));
+            assertJsonEqual(bb, cc, {
+                bb,
+                cc,
+                ii,
+                sql
+            });
         });
     }
 
+    function testDbExecWithRetryAsync() {
+// this function will test dbExecWithRetryAsync's handling-behavior
+        // test null-case handling-behavior
+        assertErrorThrownAsync(function () {
+            return dbExecWithRetryAsync({});
+        }, "invalid or closed db");
+    }
+
     async function testDbMemoryXxx() {
 // this function will test dbMemoryXxx's handling-behavior
         let data;
@@ -1480,11 +1463,15 @@ SELECT * FROM testDbExecAsync2;
         ].forEach(function ([
             rowList, rgx
         ]) {
-            assertErrorThrownAsync(function () {
-                return dbTableInsertAsync({
-                    rowList
-                });
-            }, rgx);
+            assertErrorThrownAsync(
+                dbTableInsertAsync.bind(
+                    undefined,
+                    {
+                        rowList
+                    }
+                ),
+                rgx
+            );
         });
         // test csv handling-behavior
         [
@@ -1526,17 +1513,13 @@ SELECT * FROM testDbExecAsync2;
         ].forEach(async function ([
             aa, bb
         ], ii) {
-            let cc;
-            await dbTableInsertAsync({
-                csv: aa,
-                db,
-                tableName: "csv_" + ii
-            });
-            cc = noop(
+            let cc = noop(
                 await dbExecAsync({
                     db,
                     responseType: "list",
-                    sql: "SELECT * FROM temp.csv_" + ii
+                    sql: `SELECT * FROM temp.csv_${ii}`,
+                    tmpCsv: aa,
+                    tmpTableName: "csv_" + ii
                 })
             )[0];
             assertOrThrow(
@@ -1590,7 +1573,7 @@ SELECT * FROM testDbExecAsync2;
                 return;
             }
             ii *= 0.5;
-            sql = "SELECT throwerror(" + sql + ")";
+            sql = `SELECT throwerror(${sql})`;
             assertErrorThrownAsync(function () {
                 return dbExecAsync({
                     db,
@@ -1653,6 +1636,101 @@ SELECT * FROM testDbExecAsync2;
         });
     }
 
+    async function testSqlKthpercentile() {
+// this function will test sql-kthpercentile's handling-behavior
+        let db = await dbOpenAsync({
+            filename: ":memory:"
+        });
+        [
+            [
+                [], -99, 1
+            ], [
+                [], 0, 1
+            ], [
+                [], 0.125, 1
+            ], [
+                [], 0.25, 2
+            ], [
+                [], 0.375, 3
+            ], [
+                [], 0.5, 4
+            ], [
+                [], 0.625, 5
+            ], [
+                [], 0.75, 6
+            ], [
+                [], 0.875, 7
+            ], [
+                [], 1, 8
+            ], [
+                [], 99, 8
+            ], [
+                [
+                    0.5
+                ], 0, 0.5
+            ], [
+                [
+                    0.5
+                ], 0.125, 0.5
+            ], [
+                [
+                    1.5
+                ], 0.25, 1.5
+            ], [
+                [
+                    2.5
+                ], 0.375, 2.5
+            ], [
+                [
+                    3.5
+                ], 0.5, 3.5
+            ], [
+                [
+                    4.5
+                ], 0.625, 4.5
+            ], [
+                [
+                    5.5
+                ], 0.75, 5.5
+            ], [
+                [
+                    6.5
+                ], 0.875, 6.5
+            ], [
+                [
+                    7.5
+                ], 1, 8
+            ]
+        ].forEach(async function ([
+            data, kk, expected
+        ], ii) {
+            let actual;
+            data = data.concat([
+                undefined, undefined, 8, 7, 6, 5, 4, 3, 2, 1, undefined
+            ]);
+            actual = noop(
+                await dbExecAsync({
+                    db,
+                    sql: (`
+SELECT kthpercentile(val, ${kk}) AS val FROM __tmp${ii};
+-- test null-case handling-behavior
+SELECT kthpercentile(val, ${kk}) AS val FROM __tmp${ii} WHERE 0;
+                    `),
+                    tmpRowList: data.map(function (val) {
+                        return {
+                            val
+                        };
+                    }),
+                    tmpTableName: `__tmp${ii}`
+                })
+            )[0][0].val;
+            assertJsonEqual(actual, expected, {
+                data,
+                kk
+            });
+        });
+    }
+
     addon = requireCjs(
         "./_binary_sqlmath"
         + "_napi8"
@@ -1666,11 +1744,13 @@ SELECT * FROM testDbExecAsync2;
         testDbBind,
         testDbCloseAsync,
         testDbExecAsync,
+        testDbExecWithRetryAsync,
         testDbMemoryXxx,
         testDbOpenAsync,
         testDbTableInsertAsync,
         testSqlError,
-        testSqlExt
+        testSqlExt,
+        testSqlKthpercentile
     ];
     Object.assign(local, {
         SQLITE_MAX_LENGTH2,
@@ -1699,15 +1779,21 @@ SELECT * FROM testDbExecAsync2;
         assertNumericalEqual,
         dbCloseAsync,
         dbExecAsync,
+        dbExecWithRetryAsync,
         dbGetLastBlobAsync,
         dbMemoryLoadAsync,
         dbMemorySaveAsync,
         dbOpenAsync,
         dbTableInsertAsync,
         debugInline,
+        objectDeepCopyWithKeysSorted,
         testAll,
         testList
     });
+    if (process.env.npm_config_mode_test) {
+        // mock consoleError
+        consoleError = noop;
+    }
 }());
 
 export default Object.freeze(local);

package/sqlmath_custom.mjs

@@ -0,0 +1,4 @@
+/*jslint node, beta*/
+import sqlmath from "./sqlmath.mjs";
+let exportDict = {};
+export default Object.freeze(Object.assign({}, sqlmath, exportDict));