sqlmath 2026.3.31 → 2026.4.31
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/.npmignore +1 -2
- package/CHANGELOG.md +15 -1
- package/README.md +557 -35
- package/_sqlmath.napi6_darwin_arm64.node +0 -0
- package/_sqlmath.napi6_darwin_x64.node +0 -0
- package/_sqlmath.napi6_linux_x64.node +0 -0
- package/_sqlmath.napi6_win32_x64.node +0 -0
- package/_sqlmath.shell_darwin_arm64 +0 -0
- package/_sqlmath.shell_darwin_x64 +0 -0
- package/_sqlmath.shell_linux_x64 +0 -0
- package/_sqlmath.shell_win32_x64.exe +0 -0
- package/jslint.mjs +1 -1
- package/package.json +2 -2
- package/sqlmath/lib_lightgbm_darwin_arm64.dylib +0 -0
- package/sqlmath/lib_lightgbm_darwin_x64.dylib +0 -0
- package/sqlmath/lib_lightgbm_linux_x64.so +0 -0
- package/sqlmath/lib_lightgbm_win32_x64.dll +0 -0
- package/sqlmath.mjs +7 -10
- package/sqlmath_browser.mjs +6 -6
- package/sqlmath_wasm.js +1 -1
- package/_sqlmath.shell_win32_x64.exp +0 -0
- package/_sqlmath.shell_win32_x64.lib +0 -0
- package/sqlmath_custom.mjs +0 -4
package/.npmignore
CHANGED
package/CHANGELOG.md
CHANGED
|
@@ -1,11 +1,25 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
3
|
# Todo
|
|
4
|
-
- sqlmath -
|
|
4
|
+
- sqlmath - Streamline and optimize sql-function WIN_SINEFIT2().
|
|
5
5
|
- sqlite - Add similar error-handling from builtin-sql-function PERCENTILE() into custom-sql-function QUANTILE().
|
|
6
6
|
- none
|
|
7
7
|
|
|
8
|
+
# v2026.4.31
|
|
9
|
+
- sqlmath-npm - bugfix - Fix missing lib_lightgbm pre-built libraries in npm-package. - part3 - ./sqlmath/lib_lightgbm_xxx.
|
|
10
|
+
- sqlmath-npm - bugfix - Fix missing lib_lightgbm pre-built libraries in npm-package. - part2 - git checkout origin/artifact branch-beta/
|
|
11
|
+
|
|
12
|
+
# v2026.4.30
|
|
13
|
+
- sqlmath-python - Add context manager and Pythonic API (execute, close) to SqlmathDb matching sqlite3 conventions.
|
|
14
|
+
- sqlmath-python - Add __repr__ and __bool__ methods to SqlmathDb for better debugging experience.
|
|
15
|
+
- sqlmath - Remove under-used custom-files sqlmath_custom.c, sqlmath_custom.mjs.
|
|
16
|
+
- github-ci - Pin various github-runner-os to stable/lts version.
|
|
17
|
+
- sqlmath-demo - Change historical charts from 5-years to 3-years.
|
|
18
|
+
- sqlmath-npm - bugfix - Fix missing lib_lightgbm pre-built libraries in npm-package.
|
|
19
|
+
- sqlmath-doc - Rewrite README with examples, API reference, use cases.
|
|
20
|
+
|
|
8
21
|
# v2026.3.31
|
|
22
|
+
- python-ci - bugfix - Fix shell-function shCiBaseCustom() always skipping python setup.py build_ext.
|
|
9
23
|
- ci - Speed-up windows-ci by replacing windows-latest with windows-2022.
|
|
10
24
|
- sqlmath-ci - Speed-up ci with env-var npm_config_mode_test_nopython, allowing shell-function shCiTestNodejs() to be run in background, parallel to python -m cibuildwheel.
|
|
11
25
|
- sqlmath-ci - Rename pre-built-binaries lib_lightgbm.xxx, libomp.xxx to lib_lightgbm_platform_arch.xxx, libomp_platform_arch.xxx, to prevent name-collision under darwin_arm64 and darwin_x64 builds.
|
package/README.md
CHANGED
|
@@ -1,8 +1,15 @@
|
|
|
1
|
-
# sqlmath
|
|
1
|
+
# sqlmath
|
|
2
2
|
|
|
3
|
+
**SQLite with data-science superpowers.** Run statistics, machine learning, and analytics directly in SQL queries — from Python, JavaScript, or the browser.
|
|
4
|
+
|
|
5
|
+
[](https://pypi.org/project/sqlmath/)
|
|
6
|
+
[](https://www.npmjs.com/package/sqlmath)<!--no-validate-->
|
|
7
|
+
[](https://sqlmath.github.io/sqlmath/index.html)
|
|
8
|
+
[](https://sqlmath.github.io/sqlmath/apidoc.html)
|
|
3
9
|
|
|
4
10
|
# Status
|
|
5
|
-
|
|
11
|
+
|
|
12
|
+
| Branch | [master<br>(v2026.4.31)](https://github.com/sqlmath/sqlmath/tree/master) | [beta<br>(Web Demo)](https://github.com/sqlmath/sqlmath/tree/beta) | [alpha<br>(Development)](https://github.com/sqlmath/sqlmath/tree/alpha) |
|
|
6
13
|
|--:|:--:|:--:|:--:|
|
|
7
14
|
| CI | [](https://github.com/sqlmath/sqlmath/actions?query=branch%3Amaster) | [](https://github.com/sqlmath/sqlmath/actions?query=branch%3Abeta) | [](https://github.com/sqlmath/sqlmath/actions?query=branch%3Aalpha) |
|
|
8
15
|
| Coverage | [](https://sqlmath.github.io/sqlmath/branch-master/.artifact/coverage/index.html) | [](https://sqlmath.github.io/sqlmath/branch-beta/.artifact/coverage/index.html) | [](https://sqlmath.github.io/sqlmath/branch-alpha/.artifact/coverage/index.html) |
|
|
@@ -15,22 +22,57 @@
|
|
|
15
22
|
|
|
16
23
|
1. [Web Demo](#web-demo)
|
|
17
24
|
|
|
18
|
-
2. [
|
|
19
|
-
|
|
25
|
+
2. [Why sqlmath?](#why-sqlmath)
|
|
26
|
+
|
|
27
|
+
3. [Quickstart](#quickstart)
|
|
28
|
+
- [Python](#python)
|
|
29
|
+
- [JavaScript (Node.js)](#javascript-nodejs)
|
|
30
|
+
- [Browser (WebAssembly)](#browser-webassembly)
|
|
31
|
+
|
|
32
|
+
4. [Built-in SQL Functions](#built-in-sql-functions)
|
|
33
|
+
- [Math](#math)
|
|
34
|
+
- [Statistics (Aggregate)](#statistics-aggregate)
|
|
35
|
+
- [Statistics (Scalar)](#statistics-scalar)
|
|
36
|
+
- [Arrays](#arrays)
|
|
37
|
+
- [Date/Time](#datetime)
|
|
38
|
+
- [Time Series / Signal Processing](#time-series--signal-processing)
|
|
39
|
+
- [Compression](#compression)
|
|
40
|
+
- [Type Casting](#type-casting)
|
|
41
|
+
- [Cryptography](#cryptography)
|
|
42
|
+
|
|
43
|
+
5. [Machine Learning with LightGBM](#machine-learning-with-lightgbm)
|
|
44
|
+
- [Training from a Table](#training-from-a-table)
|
|
45
|
+
- [Prediction](#prediction)
|
|
46
|
+
- [Real-World Example: Credit Card Fraud Detection](#real-world-example-credit-card-fraud-detection)
|
|
47
|
+
- [LightGBM Functions](#lightgbm-functions)
|
|
48
|
+
|
|
49
|
+
6. [Use Cases](#use-cases)
|
|
50
|
+
- [Financial Data Analysis](#financial-data-analysis)
|
|
51
|
+
- [Embedded ML Pipelines](#embedded-ml-pipelines)
|
|
52
|
+
- [Data Compression & Hashing](#data-compression--hashing)
|
|
20
53
|
|
|
21
|
-
|
|
22
|
-
- [To serve your own sqlmath website:](#to-serve-your-own-sqlmath-website)
|
|
54
|
+
7. [Why sqlmath vs Alternatives?](#why-sqlmath-vs-alternatives)
|
|
23
55
|
|
|
24
|
-
|
|
25
|
-
- [
|
|
56
|
+
8. [Python API Reference](#python-api-reference)
|
|
57
|
+
- [db_exec()](#db_exec)
|
|
26
58
|
|
|
27
|
-
|
|
59
|
+
9. [Node.js API Reference](#nodejs-api-reference)
|
|
28
60
|
|
|
29
|
-
|
|
61
|
+
10. [Platform Notes](#platform-notes)
|
|
30
62
|
|
|
31
|
-
|
|
63
|
+
11. [Building from Source](#building-from-source)
|
|
64
|
+
- [Prerequisites](#prerequisites)
|
|
65
|
+
- [Build](#build)
|
|
66
|
+
- [Run Tests](#run-tests)
|
|
67
|
+
- [Serve Demo Locally](#serve-demo-locally)
|
|
32
68
|
|
|
33
|
-
|
|
69
|
+
12. [Package Listing](#package-listing)
|
|
70
|
+
|
|
71
|
+
13. [Changelog](#changelog)
|
|
72
|
+
|
|
73
|
+
14. [License](#license)
|
|
74
|
+
|
|
75
|
+
15. [Devops Instruction](#devops-instruction)
|
|
34
76
|
- [python pypi publish](#python-pypi-publish)
|
|
35
77
|
- [sqlite upgrade](#sqlite-upgrade)
|
|
36
78
|
|
|
@@ -43,53 +85,533 @@
|
|
|
43
85
|
|
|
44
86
|
|
|
45
87
|
<br><br>
|
|
46
|
-
#
|
|
88
|
+
# Why sqlmath?
|
|
89
|
+
|
|
90
|
+
SQLite is everywhere — it's the most deployed database in the world. But it lacks the statistical and ML functions needed for data-science. sqlmath fixes that.
|
|
91
|
+
|
|
92
|
+
**What you get:**
|
|
93
|
+
- **20+ built-in functions** for math, statistics, arrays, dates, and compression
|
|
94
|
+
- **LightGBM integration** — train and predict ML models directly in SQL
|
|
95
|
+
- **Zero dependencies** — single binary, no external libraries needed
|
|
96
|
+
- **Multi-platform** — Python, Node.js, browser (WebAssembly)
|
|
97
|
+
|
|
98
|
+
|
|
99
|
+
<br><br>
|
|
100
|
+
# Quickstart
|
|
47
101
|
|
|
48
102
|
|
|
49
103
|
<br><br>
|
|
50
|
-
###
|
|
104
|
+
### Python
|
|
105
|
+
|
|
51
106
|
```shell
|
|
52
|
-
|
|
107
|
+
pip install sqlmath
|
|
108
|
+
```
|
|
53
109
|
|
|
54
|
-
|
|
55
|
-
|
|
56
|
-
|
|
110
|
+
```python
|
|
111
|
+
from sqlmath import db_open, db_exec, db_close
|
|
112
|
+
|
|
113
|
+
# Open an in-memory database
|
|
114
|
+
db = db_open(":memory:")
|
|
115
|
+
|
|
116
|
+
# Create sample data
|
|
117
|
+
db_exec(db=db, sql="""
|
|
118
|
+
CREATE TABLE prices (symbol TEXT, price REAL);
|
|
119
|
+
INSERT INTO prices VALUES ('AAPL', 150.0), ('GOOGL', 140.0), ('MSFT', 380.0);
|
|
120
|
+
""")
|
|
121
|
+
|
|
122
|
+
# Use built-in statistical functions
|
|
123
|
+
result = db_exec(db=db, sql="""
|
|
124
|
+
SELECT
|
|
125
|
+
AVG(price) AS mean_price,
|
|
126
|
+
MEDIAN2(price) AS median_price,
|
|
127
|
+
STDEV(price) AS stdev_price
|
|
128
|
+
FROM prices
|
|
129
|
+
""")
|
|
130
|
+
print(result)
|
|
131
|
+
# [[{'mean_price': 223.33, 'median_price': 150.0, 'stdev_price': 135.77}]]
|
|
132
|
+
|
|
133
|
+
db_close(db)
|
|
134
|
+
```
|
|
57
135
|
|
|
58
|
-
# build nodejs binary ./_binary_sqlmath_napi8_xxx_x64.node
|
|
59
|
-
npm run test2
|
|
60
136
|
|
|
61
|
-
|
|
62
|
-
|
|
137
|
+
<br><br>
|
|
138
|
+
### JavaScript (Node.js)
|
|
139
|
+
|
|
140
|
+
```shell
|
|
141
|
+
npm install sqlmath
|
|
142
|
+
```
|
|
143
|
+
|
|
144
|
+
```javascript
|
|
145
|
+
import { dbOpenAsync, dbExecAsync, dbCloseAsync } from "sqlmath";
|
|
146
|
+
|
|
147
|
+
const db = await dbOpenAsync({ filename: ":memory:" });
|
|
148
|
+
|
|
149
|
+
await dbExecAsync({
|
|
150
|
+
db,
|
|
151
|
+
sql: `
|
|
152
|
+
CREATE TABLE prices (symbol TEXT, price REAL);
|
|
153
|
+
INSERT INTO prices VALUES ('AAPL', 150.0), ('GOOGL', 140.0), ('MSFT', 380.0);
|
|
154
|
+
`
|
|
155
|
+
});
|
|
156
|
+
|
|
157
|
+
const result = await dbExecAsync({
|
|
158
|
+
db,
|
|
159
|
+
sql: "SELECT AVG(price) AS mean_price FROM prices"
|
|
160
|
+
});
|
|
161
|
+
console.log(result);
|
|
162
|
+
|
|
163
|
+
await dbCloseAsync(db);
|
|
63
164
|
```
|
|
64
165
|
|
|
65
166
|
|
|
66
167
|
<br><br>
|
|
67
|
-
|
|
168
|
+
### Browser (WebAssembly)
|
|
169
|
+
|
|
170
|
+
Try it live: **[sqlmath.github.io/sqlmath](https://sqlmath.github.io/sqlmath/index.html)**
|
|
68
171
|
|
|
69
172
|
|
|
70
173
|
<br><br>
|
|
71
|
-
|
|
72
|
-
```shell
|
|
73
|
-
#!/bin/sh
|
|
174
|
+
# Built-in SQL Functions
|
|
74
175
|
|
|
75
|
-
# cd <sqlmath repo>
|
|
76
176
|
|
|
77
|
-
|
|
78
|
-
|
|
177
|
+
<br><br>
|
|
178
|
+
### Math
|
|
179
|
+
|
|
180
|
+
| Function | Description |
|
|
181
|
+
|----------|-------------|
|
|
182
|
+
| `cot(x)` | Cotangent |
|
|
183
|
+
| `coth(x)` | Hyperbolic cotangent |
|
|
184
|
+
| `fmod(x, y)` | Floating-point modulo |
|
|
185
|
+
| `squared(x)` | Square of x |
|
|
186
|
+
| `squaredwithsign(x)` | Square preserving sign (negative input → negative output) |
|
|
187
|
+
| `sqrtwithsign(x)` | Square root preserving sign |
|
|
188
|
+
| `normalizewithsqrt(x)` | Normalize using square root |
|
|
189
|
+
| `normalizewithsquared(x)` | Normalize using square |
|
|
190
|
+
|
|
191
|
+
|
|
192
|
+
<br><br>
|
|
193
|
+
### Statistics (Aggregate)
|
|
194
|
+
|
|
195
|
+
| Function | Description |
|
|
196
|
+
|----------|-------------|
|
|
197
|
+
| `MEDIAN2(x)` | Median value (aggregate) |
|
|
198
|
+
| `PERCENTILE(x, p)` | Percentile at p (aggregate) |
|
|
199
|
+
| `QUANTILE(x, q)` | Quantile at q (aggregate) |
|
|
200
|
+
| `STDEV(x)` | Standard deviation (sample, window-capable) |
|
|
201
|
+
|
|
202
|
+
|
|
203
|
+
<br><br>
|
|
204
|
+
### Statistics (Scalar)
|
|
205
|
+
|
|
206
|
+
| Function | Description |
|
|
207
|
+
|----------|-------------|
|
|
208
|
+
| `marginoferror95(p, n)` | 95% margin of error: sqrt(p*(1-p)/n) |
|
|
209
|
+
| `random1()` | Random float in [0, 1) |
|
|
210
|
+
|
|
211
|
+
|
|
212
|
+
<br><br>
|
|
213
|
+
### Arrays
|
|
214
|
+
|
|
215
|
+
| Function | Description |
|
|
216
|
+
|----------|-------------|
|
|
217
|
+
| `doublearray_array(...)` | Create double array from values |
|
|
218
|
+
| `doublearray_extract(arr, i)` | Extract element at index |
|
|
219
|
+
| `doublearray_jsonfrom(json)` | Create array from JSON |
|
|
220
|
+
| `doublearray_jsonto(arr)` | Convert array to JSON |
|
|
221
|
+
|
|
222
|
+
|
|
223
|
+
<br><br>
|
|
224
|
+
### Date/Time
|
|
225
|
+
|
|
226
|
+
sqlmath extends SQLite's date functions with integer-format conversions (YYYYMMDDHHMMSS).
|
|
227
|
+
|
|
228
|
+
|
|
229
|
+
<br><br>
|
|
230
|
+
### Time Series / Signal Processing
|
|
231
|
+
|
|
232
|
+
| Function | Description |
|
|
233
|
+
|----------|-------------|
|
|
234
|
+
| `WIN_SINEFIT2(...)` | Fit sine wave to data (aggregate window function) |
|
|
235
|
+
| `SINEFIT_REFITLAST(...)` | Refit sine wave with new data point |
|
|
236
|
+
|
|
237
|
+
|
|
238
|
+
<br><br>
|
|
239
|
+
### Compression
|
|
240
|
+
|
|
241
|
+
| Function | Description |
|
|
242
|
+
|----------|-------------|
|
|
243
|
+
| `GZIP_COMPRESS(blob)` | Compress BLOB with gzip |
|
|
244
|
+
| `GZIP_UNCOMPRESS(blob)` | Decompress gzip BLOB |
|
|
245
|
+
|
|
246
|
+
> **Note:** Input must be BLOB. Use `CAST(text AS BLOB)` for text data.
|
|
247
|
+
|
|
248
|
+
|
|
249
|
+
<br><br>
|
|
250
|
+
### Type Casting
|
|
251
|
+
|
|
252
|
+
| Function | Description |
|
|
253
|
+
|----------|-------------|
|
|
254
|
+
| `castrealornull(x)` | Cast to REAL or NULL |
|
|
255
|
+
| `castrealorzero(x)` | Cast to REAL or 0.0 |
|
|
256
|
+
| `casttextorempty(x)` | Cast to TEXT or '' |
|
|
257
|
+
| `roundorzero(x, n)` | Round or return 0 |
|
|
258
|
+
|
|
259
|
+
|
|
260
|
+
<br><br>
|
|
261
|
+
### Cryptography
|
|
262
|
+
|
|
263
|
+
| Function | Description |
|
|
264
|
+
|----------|-------------|
|
|
265
|
+
| `sha256(data)` | SHA-256 hash (returns BLOB) |
|
|
266
|
+
|
|
267
|
+
```sql
|
|
268
|
+
SELECT HEX(sha256('hello')) AS hash;
|
|
269
|
+
-- 2CF24DBA5FB0A30E26E83B2AC5B9E29E1B161E5C1FA7425E73043362938B9824
|
|
270
|
+
```
|
|
271
|
+
|
|
272
|
+
|
|
273
|
+
<br><br>
|
|
274
|
+
# Machine Learning with LightGBM
|
|
275
|
+
|
|
276
|
+
sqlmath embeds [LightGBM](https://lightgbm.readthedocs.io/) for gradient boosting directly in SQL queries. Train models on your data without leaving SQL — no data shuffle to Python needed.
|
|
277
|
+
|
|
278
|
+
> **Python users:** You must call `lgbm_dlopen()` before using any `lgbm_*` functions. See [Platform Notes](#platform-notes) for details.
|
|
279
|
+
|
|
280
|
+
|
|
281
|
+
<br><br>
|
|
282
|
+
### Training from a Table
|
|
283
|
+
|
|
284
|
+
`LGBM_TRAINFROMTABLE` is an **aggregate function** — it consumes rows like `SUM()` or `AVG()`, but outputs a trained model BLOB.
|
|
285
|
+
|
|
286
|
+
```sql
|
|
287
|
+
-- Create a table to store the model
|
|
288
|
+
CREATE TABLE model_store (model BLOB);
|
|
289
|
+
|
|
290
|
+
-- Train directly from your data table
|
|
291
|
+
INSERT INTO model_store(model)
|
|
292
|
+
SELECT
|
|
293
|
+
LGBM_TRAINFROMTABLE(
|
|
294
|
+
-- Training parameters
|
|
295
|
+
(
|
|
296
|
+
'objective=binary'
|
|
297
|
+
|| ' learning_rate=0.1'
|
|
298
|
+
|| ' metric=auc'
|
|
299
|
+
|| ' num_leaves=31'
|
|
300
|
+
|| ' verbosity=0'
|
|
301
|
+
),
|
|
302
|
+
50, -- num_iterations
|
|
303
|
+
10, -- eval_step (early stopping check interval)
|
|
304
|
+
'max_bin=15', -- data parameters
|
|
305
|
+
NULL, -- reference dataset (NULL for training set)
|
|
306
|
+
-- Columns: first column MUST be the label
|
|
307
|
+
label, feature1, feature2, feature3, feature4
|
|
308
|
+
)
|
|
309
|
+
FROM training_data;
|
|
310
|
+
```
|
|
311
|
+
|
|
312
|
+
|
|
313
|
+
<br><br>
|
|
314
|
+
### Prediction
|
|
315
|
+
|
|
316
|
+
```sql
|
|
317
|
+
-- Predict on new data using the stored model
|
|
318
|
+
SELECT
|
|
319
|
+
id,
|
|
320
|
+
LGBM_PREDICTFORTABLE(
|
|
321
|
+
(SELECT model FROM model_store),
|
|
322
|
+
0, -- predict_type (0 = normal probability)
|
|
323
|
+
0, -- start_iteration
|
|
324
|
+
50, -- num_iterations
|
|
325
|
+
'', -- prediction parameters
|
|
326
|
+
feature1, feature2, feature3, feature4
|
|
327
|
+
) AS prediction
|
|
328
|
+
FROM test_data;
|
|
329
|
+
```
|
|
330
|
+
|
|
331
|
+
|
|
332
|
+
<br><br>
|
|
333
|
+
### Real-World Example: Credit Card Fraud Detection
|
|
334
|
+
|
|
335
|
+
From the [Kaggle notebook](https://www.kaggle.com/code/kaizhu256/sql-is-all-you-need) — training on 284,807 transactions with 0.17% fraud rate:
|
|
336
|
+
|
|
337
|
+
```python
|
|
338
|
+
from sqlmath import db_open, db_exec, db_table_import
|
|
339
|
+
|
|
340
|
+
db = db_open("fraud.sqlite")
|
|
341
|
+
|
|
342
|
+
# Import Kaggle credit card dataset
|
|
343
|
+
db_table_import(db=db, filename="creditcard.csv", table_name="transactions", mode="csv")
|
|
344
|
+
|
|
345
|
+
# Split 80/20 train/test
|
|
346
|
+
db_exec(db=db, sql="""
|
|
347
|
+
CREATE TABLE train AS SELECT * FROM transactions WHERE RANDOM() % 5 != 0;
|
|
348
|
+
CREATE TABLE test AS SELECT * FROM transactions WHERE RANDOM() % 5 = 0;
|
|
349
|
+
""")
|
|
350
|
+
|
|
351
|
+
# Train with automatic class imbalance handling
|
|
352
|
+
db_exec(db=db, sql="""
|
|
353
|
+
CREATE TABLE model_store (model BLOB);
|
|
354
|
+
|
|
355
|
+
INSERT INTO model_store(model)
|
|
356
|
+
SELECT
|
|
357
|
+
LGBM_TRAINFROMTABLE(
|
|
358
|
+
(
|
|
359
|
+
'objective=binary'
|
|
360
|
+
|| ' learning_rate=0.05'
|
|
361
|
+
|| ' metric=auc'
|
|
362
|
+
|| ' is_unbalance=true' -- Auto-weight minority class
|
|
363
|
+
|| ' num_leaves=31'
|
|
364
|
+
|| ' verbosity=0'
|
|
365
|
+
),
|
|
366
|
+
100, -- num_iterations
|
|
367
|
+
10, -- eval_step
|
|
368
|
+
'max_bin=255', -- data_params
|
|
369
|
+
NULL, -- reference
|
|
370
|
+
-- First column = label, rest = features (V1-V28 + Amount)
|
|
371
|
+
Class, V1, V2, V3, V4, V5, V6, V7, V8, V9, V10,
|
|
372
|
+
V11, V12, V13, V14, V15, V16, V17, V18, V19, V20,
|
|
373
|
+
V21, V22, V23, V24, V25, V26, V27, V28, Amount
|
|
374
|
+
)
|
|
375
|
+
FROM train;
|
|
376
|
+
""")
|
|
79
377
|
```
|
|
80
378
|
|
|
81
379
|
|
|
82
380
|
<br><br>
|
|
83
|
-
|
|
381
|
+
### LightGBM Functions
|
|
382
|
+
|
|
383
|
+
| Function | Type | Description |
|
|
384
|
+
|----------|------|-------------|
|
|
385
|
+
| `lgbm_dlopen(path)` | scalar | Load LightGBM library (required in Python) |
|
|
386
|
+
| `LGBM_TRAINFROMTABLE(params, n_iter, eval, data_params, ref, label, ...)` | aggregate | Train model from table columns |
|
|
387
|
+
| `LGBM_TRAINFROMFILE(params, n_iter, eval, train_file, data_params, test_file)` | scalar | Train from LibSVM file |
|
|
388
|
+
| `LGBM_PREDICTFORTABLE(model, type, start, n, params, ...)` | window | Predict per row |
|
|
389
|
+
| `LGBM_PREDICTFORFILE(model, type, start, n, params, file, header, out)` | scalar | Predict and save to file |
|
|
390
|
+
| `lgbm_extract(result, key)` | scalar | Extract value from result |
|
|
391
|
+
|
|
392
|
+
**Complete Example:**
|
|
393
|
+
|
|
394
|
+
[](https://www.kaggle.com/code/kaizhu256/sql-is-all-you-need)
|
|
395
|
+
|
|
396
|
+
Full notebook with fraud detection, intraday trading signals, and model persistence.
|
|
397
|
+
|
|
398
|
+
**Kaggle Environment:**
|
|
399
|
+
- Linux x64, Python 3.12, Node.js 24
|
|
400
|
+
- Datasets: [Credit Card Fraud](https://www.kaggle.com/datasets/mlg-ulb/creditcardfraud) (284K transactions), SPY intraday OHLCV
|
|
401
|
+
- sqlmath installs via `pip install sqlmath==2026.4.31`
|
|
402
|
+
|
|
403
|
+
|
|
404
|
+
<br><br>
|
|
405
|
+
# Use Cases
|
|
406
|
+
|
|
407
|
+
|
|
408
|
+
<br><br>
|
|
409
|
+
### Financial Data Analysis
|
|
410
|
+
|
|
411
|
+
```python
|
|
412
|
+
from sqlmath import db_open, db_exec, db_close, db_table_import
|
|
413
|
+
|
|
414
|
+
db = db_open(":memory:")
|
|
415
|
+
|
|
416
|
+
# Import OHLCV data
|
|
417
|
+
db_table_import(db=db, table_name="prices", filename="prices.csv", mode="csv")
|
|
418
|
+
|
|
419
|
+
# Calculate rolling statistics in SQL
|
|
420
|
+
result = db_exec(db=db, sql="""
|
|
421
|
+
SELECT
|
|
422
|
+
date,
|
|
423
|
+
close,
|
|
424
|
+
AVG(close) OVER (ORDER BY date ROWS 20 PRECEDING) AS sma_20,
|
|
425
|
+
STDEV(close) OVER (ORDER BY date ROWS 20 PRECEDING) AS volatility,
|
|
426
|
+
PERCENTILE(volume, 50) OVER (ORDER BY date ROWS 5 PRECEDING) AS median_volume
|
|
427
|
+
FROM prices
|
|
428
|
+
ORDER BY date DESC
|
|
429
|
+
LIMIT 10
|
|
430
|
+
""")
|
|
431
|
+
|
|
432
|
+
db_close(db)
|
|
433
|
+
```
|
|
84
434
|
|
|
85
435
|
|
|
86
436
|
<br><br>
|
|
87
|
-
###
|
|
437
|
+
### Embedded ML Pipelines
|
|
438
|
+
|
|
439
|
+
Train and deploy models without data movement:
|
|
440
|
+
|
|
441
|
+
```python
|
|
442
|
+
# 1. Load data into SQLite
|
|
443
|
+
db_table_import(db=db, table_name="features", filename="training_data.csv", mode="csv")
|
|
444
|
+
|
|
445
|
+
# 2. Train model in-database
|
|
446
|
+
db_exec(db=db, sql="""
|
|
447
|
+
CREATE TABLE models AS
|
|
448
|
+
SELECT LGBM_TRAINFROMTABLE(...) AS model FROM features
|
|
449
|
+
""")
|
|
450
|
+
|
|
451
|
+
# 3. Score new data in-database
|
|
452
|
+
db_exec(db=db, sql="""
|
|
453
|
+
SELECT id, LGBM_PREDICTFORTABLE(model, ...) AS score
|
|
454
|
+
FROM new_data, models
|
|
455
|
+
""")
|
|
456
|
+
|
|
457
|
+
# 4. Export results
|
|
458
|
+
db_file_save(db=db, filename="scored_data.sqlite")
|
|
459
|
+
```
|
|
460
|
+
|
|
461
|
+
|
|
462
|
+
<br><br>
|
|
463
|
+
### Data Compression & Hashing
|
|
464
|
+
|
|
465
|
+
```sql
|
|
466
|
+
-- Compress large text fields
|
|
467
|
+
UPDATE documents SET
|
|
468
|
+
content_compressed = GZIP_COMPRESS(CAST(content AS BLOB));
|
|
469
|
+
|
|
470
|
+
-- Generate content hashes for deduplication
|
|
471
|
+
SELECT HEX(sha256(content)) AS content_hash, COUNT(*)
|
|
472
|
+
FROM documents
|
|
473
|
+
GROUP BY content_hash
|
|
474
|
+
HAVING COUNT(*) > 1;
|
|
475
|
+
```
|
|
476
|
+
|
|
477
|
+
|
|
478
|
+
<br><br>
|
|
479
|
+
# Why sqlmath vs Alternatives?
|
|
480
|
+
|
|
481
|
+
| Feature | sqlmath | pandas | DuckDB | sqlite3 |
|
|
482
|
+
|---------|---------|--------|--------|---------|
|
|
483
|
+
| In-database ML | ✅ LightGBM | ❌ | ❌ | ❌ |
|
|
484
|
+
| Zero data shuffle | ✅ | ❌ | ✅ | ✅ |
|
|
485
|
+
| Statistical functions | ✅ 20+ | ✅ | ✅ | ❌ |
|
|
486
|
+
| Browser support | ✅ WASM | ❌ | ❌ | ❌ |
|
|
487
|
+
| Single file deployment | ✅ | ❌ | ✅ | ✅ |
|
|
488
|
+
| Memory efficiency | ✅ Streaming | ❌ In-memory | ✅ | ✅ |
|
|
489
|
+
|
|
490
|
+
**Choose sqlmath when:**
|
|
491
|
+
- You need ML training/inference inside the database
|
|
492
|
+
- Your data is already in SQLite
|
|
493
|
+
- You want browser-based analytics (WebAssembly)
|
|
494
|
+
- You need statistical functions beyond basic SQL
|
|
495
|
+
|
|
496
|
+
**Choose pandas when:**
|
|
497
|
+
- You need complex data transformations
|
|
498
|
+
- Interactive exploration with rich visualization
|
|
499
|
+
- Your workflow is Python-centric
|
|
500
|
+
|
|
501
|
+
**Choose DuckDB when:**
|
|
502
|
+
- You need fast analytical queries on large datasets
|
|
503
|
+
- You want pandas DataFrame interop
|
|
504
|
+
- OLAP workloads are primary
|
|
505
|
+
|
|
506
|
+
|
|
507
|
+
<br><br>
|
|
508
|
+
# Python API Reference
|
|
509
|
+
|
|
510
|
+
```python
|
|
511
|
+
from sqlmath import (
|
|
512
|
+
db_open, # Open database
|
|
513
|
+
db_close, # Close database
|
|
514
|
+
db_exec, # Execute SQL, return results
|
|
515
|
+
db_exec_and_return_lastblob, # Execute SQL, return last blob
|
|
516
|
+
db_file_load, # Load database from file
|
|
517
|
+
db_file_save, # Save database to file
|
|
518
|
+
db_table_import, # Import data into table
|
|
519
|
+
db_noop, # No-op for testing
|
|
520
|
+
)
|
|
521
|
+
```
|
|
522
|
+
|
|
523
|
+
|
|
524
|
+
<br><br>
|
|
525
|
+
### db_exec()
|
|
526
|
+
|
|
527
|
+
```python
|
|
528
|
+
result = db_exec(
|
|
529
|
+
db=db, # Database connection
|
|
530
|
+
sql="SELECT ...", # SQL statement(s)
|
|
531
|
+
bind_list=[...], # Bind parameters (list or dict)
|
|
532
|
+
response_type=None, # None (default) returns list-of-dicts
|
|
533
|
+
# "list" | "lastblob" | "arraybuffer"
|
|
534
|
+
)
|
|
535
|
+
# Default: [[{'col1': val1, 'col2': val2}, ...]]
|
|
536
|
+
```
|
|
537
|
+
|
|
538
|
+
|
|
539
|
+
<br><br>
|
|
540
|
+
# Node.js API Reference
|
|
88
541
|
- https://sqlmath.github.io/sqlmath/apidoc.html
|
|
89
542
|
|
|
90
543
|
[](https://sqlmath.github.io/sqlmath/apidoc.html)
|
|
91
544
|
|
|
92
545
|
|
|
546
|
+
<br><br>
|
|
547
|
+
# Platform Notes
|
|
548
|
+
|
|
549
|
+
The JavaScript (Node.js) binding is more mature than Python. Key differences:
|
|
550
|
+
|
|
551
|
+
| Feature | Node.js | Python |
|
|
552
|
+
|---------|---------|--------|
|
|
553
|
+
| Async vs sync api | **async only:** `result = await dbExecAsync({...})` | **sync only:** `result = db_exec(...)` |
|
|
554
|
+
| Connection pooling | ✅ `db = await dbOpenAsync({threadCount: 4, ...})` | ❌ `db = db_open(...)` |
|
|
555
|
+
| Type hints | N/A | ❌ Not yet |
|
|
556
|
+
| Context manager | N/A | ❌ `with db_open()` not supported |
|
|
557
|
+
|
|
558
|
+
> **Note:** The Python wrapper is a work in progress. Contributions welcome!
|
|
559
|
+
|
|
560
|
+
|
|
561
|
+
<br><br>
|
|
562
|
+
# Building from Source
|
|
563
|
+
|
|
564
|
+
|
|
565
|
+
<br><br>
|
|
566
|
+
### Prerequisites
|
|
567
|
+
|
|
568
|
+
- Node.js 24+
|
|
569
|
+
- Python 3.12+
|
|
570
|
+
- C compiler (gcc, clang, or MSVC)
|
|
571
|
+
|
|
572
|
+
|
|
573
|
+
<br><br>
|
|
574
|
+
### Build
|
|
575
|
+
|
|
576
|
+
```shell
|
|
577
|
+
#!/bin/sh
|
|
578
|
+
|
|
579
|
+
# git clone sqlmath repo
|
|
580
|
+
git clone https://github.com/sqlmath/sqlmath --branch=beta --single-branch
|
|
581
|
+
cd sqlmath
|
|
582
|
+
|
|
583
|
+
# Build native binary
|
|
584
|
+
npm run test2
|
|
585
|
+
|
|
586
|
+
# Build WebAssembly (optional)
|
|
587
|
+
sh jslint_ci.sh shCiBuildWasm
|
|
588
|
+
```
|
|
589
|
+
|
|
590
|
+
|
|
591
|
+
<br><br>
|
|
592
|
+
### Run Tests
|
|
593
|
+
|
|
594
|
+
```shell
|
|
595
|
+
# Full test suite (includes full clean-build)
|
|
596
|
+
npm run test2
|
|
597
|
+
|
|
598
|
+
# Full test suite (includes partial re-build of modified files)
|
|
599
|
+
npm run test
|
|
600
|
+
|
|
601
|
+
# Quick test (skip build)
|
|
602
|
+
npm run test --fast
|
|
603
|
+
```
|
|
604
|
+
|
|
605
|
+
|
|
606
|
+
<br><br>
|
|
607
|
+
### Serve Demo Locally
|
|
608
|
+
|
|
609
|
+
```shell
|
|
610
|
+
PORT=8080 sh jslint_ci.sh shHttpFileServer
|
|
611
|
+
# Open http://localhost:8080/index.html
|
|
612
|
+
```
|
|
613
|
+
|
|
614
|
+
|
|
93
615
|
<br><br>
|
|
94
616
|
# Package Listing
|
|
95
617
|

|
|
@@ -121,11 +643,11 @@ PORT=8080 sh jslint_ci.sh shHttpFileServer
|
|
|
121
643
|
```shell
|
|
122
644
|
python -m build
|
|
123
645
|
#
|
|
124
|
-
twine upload --repository testpypi dist/sqlmath-2026.
|
|
125
|
-
py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2026.
|
|
646
|
+
twine upload --repository testpypi dist/sqlmath-2026.4.31*
|
|
647
|
+
py -m pip install --index-url https://test.pypi.org/simple/ sqlmath==2026.4.31
|
|
126
648
|
#
|
|
127
|
-
twine upload dist/sqlmath-2026.
|
|
128
|
-
pip install sqlmath==2026.
|
|
649
|
+
twine upload dist/sqlmath-2026.4.31*
|
|
650
|
+
pip install sqlmath==2026.4.31
|
|
129
651
|
```
|
|
130
652
|
|
|
131
653
|
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/_sqlmath.shell_linux_x64
CHANGED
|
Binary file
|
|
Binary file
|
package/jslint.mjs
CHANGED
|
@@ -163,7 +163,7 @@ let jslint_charset_ascii = (
|
|
|
163
163
|
+ "@ABCDEFGHIJKLMNOPQRSTUVWXYZ[\\]^_"
|
|
164
164
|
+ "`abcdefghijklmnopqrstuvwxyz{|}~\u007f"
|
|
165
165
|
);
|
|
166
|
-
let jslint_edition = "v2026.
|
|
166
|
+
let jslint_edition = "v2026.4.30";
|
|
167
167
|
let jslint_export; // The jslint object to be exported.
|
|
168
168
|
let jslint_fudge = 1; // Fudge starting line and starting
|
|
169
169
|
// ... column to 1.
|
package/package.json
CHANGED
|
@@ -5,7 +5,7 @@
|
|
|
5
5
|
},
|
|
6
6
|
"counter": 0,
|
|
7
7
|
"description": "sqlite for data-science",
|
|
8
|
-
"fileCount":
|
|
8
|
+
"fileCount": 44,
|
|
9
9
|
"homepage": "https://github.com/sqlmath/sqlmath",
|
|
10
10
|
"keywords": [
|
|
11
11
|
"data-science",
|
|
@@ -32,5 +32,5 @@
|
|
|
32
32
|
},
|
|
33
33
|
"shCiArtifactUpload": 1,
|
|
34
34
|
"shCiPublishNpm": 1,
|
|
35
|
-
"version": "2026.
|
|
35
|
+
"version": "2026.4.31"
|
|
36
36
|
}
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
|
Binary file
|
package/sqlmath.mjs
CHANGED
|
@@ -128,7 +128,7 @@ let {
|
|
|
128
128
|
let sqlMessageDict = {}; // dict of web-worker-callbacks
|
|
129
129
|
let sqlMessageId = 0;
|
|
130
130
|
let sqlWorker;
|
|
131
|
-
let version = "v2026.
|
|
131
|
+
let version = "v2026.4.31";
|
|
132
132
|
|
|
133
133
|
async function assertErrorThrownAsync(asyncFunc, regexp) {
|
|
134
134
|
|
|
@@ -327,7 +327,6 @@ async function ciBuildExt({
|
|
|
327
327
|
(`
|
|
328
328
|
(set -e
|
|
329
329
|
# rebuild binding
|
|
330
|
-
rm -rf build/Release/obj/SRC_SQLMATH_CUSTOM/
|
|
331
330
|
node "${binNodegyp}" build --release
|
|
332
331
|
# node "${binNodegyp}" build --release --loglevel=verbose
|
|
333
332
|
mv build/Release/binding.node "${cModulePath}"
|
|
@@ -339,7 +338,7 @@ async function ciBuildExt({
|
|
|
339
338
|
rm -f ${SQLMATH_EXE}
|
|
340
339
|
python setup.py exe_link \
|
|
341
340
|
./build/Release/SRC_SQLITE_BASE.lib \
|
|
342
|
-
./build/Release/
|
|
341
|
+
./build/Release/SRC_SQLMATH_BASE.lib \
|
|
343
342
|
./build/Release/obj/shell/sqlmath_external_sqlite.obj \
|
|
344
343
|
./zlib.v1.3.1.vcpkg.x64-windows-static.lib \
|
|
345
344
|
\
|
|
@@ -424,17 +423,15 @@ SQLMATH_CFLAG_WNO_LIST=" \\
|
|
|
424
423
|
},
|
|
425
424
|
{
|
|
426
425
|
"defines": [
|
|
427
|
-
"SRC_SQLMATH_BASE_C2"
|
|
428
|
-
"SRC_SQLMATH_CUSTOM_C2"
|
|
426
|
+
"SRC_SQLMATH_BASE_C2"
|
|
429
427
|
],
|
|
430
428
|
"dependencies": [
|
|
431
429
|
"SRC_SQLITE_BASE"
|
|
432
430
|
],
|
|
433
431
|
"sources": [
|
|
434
|
-
"sqlmath_base.c"
|
|
435
|
-
"sqlmath_custom.c"
|
|
432
|
+
"sqlmath_base.c"
|
|
436
433
|
],
|
|
437
|
-
"target_name": "
|
|
434
|
+
"target_name": "SRC_SQLMATH_BASE",
|
|
438
435
|
"type": "static_library"
|
|
439
436
|
},
|
|
440
437
|
{
|
|
@@ -442,7 +439,7 @@ SQLMATH_CFLAG_WNO_LIST=" \\
|
|
|
442
439
|
"SRC_SQLMATH_NODEJS_C2"
|
|
443
440
|
],
|
|
444
441
|
"dependencies": [
|
|
445
|
-
"
|
|
442
|
+
"SRC_SQLMATH_BASE"
|
|
446
443
|
],
|
|
447
444
|
"sources": [
|
|
448
445
|
"sqlmath_base.c"
|
|
@@ -465,7 +462,7 @@ SQLMATH_CFLAG_WNO_LIST=" \\
|
|
|
465
462
|
"SRC_SQLITE_SHELL_C2"
|
|
466
463
|
],
|
|
467
464
|
"dependencies": [
|
|
468
|
-
"
|
|
465
|
+
"SRC_SQLMATH_BASE"
|
|
469
466
|
],
|
|
470
467
|
"sources": [
|
|
471
468
|
"sqlmath_external_sqlite.c"
|
package/sqlmath_browser.mjs
CHANGED
|
@@ -404,8 +404,8 @@ ALTER TABLE tradebot_tech_intra_month DROP COLUMN xdate2;
|
|
|
404
404
|
"1 month",
|
|
405
405
|
"ytd",
|
|
406
406
|
"1 year",
|
|
407
|
-
"
|
|
408
|
-
"
|
|
407
|
+
"3 year",
|
|
408
|
+
"3 year reverse timeline"
|
|
409
409
|
].map(function (dateInterval) {
|
|
410
410
|
let optionDict;
|
|
411
411
|
let tableChart;
|
|
@@ -520,8 +520,8 @@ UPDATE __tmp1
|
|
|
520
520
|
MAX(aa,
|
|
521
521
|
${
|
|
522
522
|
(
|
|
523
|
-
dateInterval === "
|
|
524
|
-
? "DATE(bb, '-
|
|
523
|
+
dateInterval === "3 year reverse timeline"
|
|
524
|
+
? "DATE(bb, '-3 YEAR')"
|
|
525
525
|
: dateInterval === "ytd"
|
|
526
526
|
? "DATE(STRFTIME('%Y', bb) || '-01-01', '-1 DAY')"
|
|
527
527
|
: "DATE(bb, '-" + dateInterval + "')"
|
|
@@ -583,7 +583,7 @@ UPDATE ${tableChart}
|
|
|
583
583
|
SET
|
|
584
584
|
${
|
|
585
585
|
(
|
|
586
|
-
dateInterval === "
|
|
586
|
+
dateInterval === "3 year reverse timeline"
|
|
587
587
|
? "yy = ROUND(100 * (1.0 / (yy * inv) - 1), 4)"
|
|
588
588
|
: "yy = ROUND(100 * (yy * inv - 1), 4)"
|
|
589
589
|
)
|
|
@@ -599,7 +599,7 @@ UPDATE ${tableChart}
|
|
|
599
599
|
PARTITION BY series_index ORDER BY xx
|
|
600
600
|
${
|
|
601
601
|
(
|
|
602
|
-
dateInterval === "
|
|
602
|
+
dateInterval === "3 year reverse timeline"
|
|
603
603
|
? "DESC"
|
|
604
604
|
: "ASC"
|
|
605
605
|
)
|
package/sqlmath_wasm.js
CHANGED
|
Binary file
|
|
Binary file
|
package/sqlmath_custom.mjs
DELETED