sensemaking 0.23.0 → 0.24.0

package/dist/cjs/store/turso/connection.js

@@ -9,14 +9,28 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get cacheFilePath () {
+        return cacheFilePath;
+    },
     get checkpointWal () {
         return checkpointWal;
     },
     get createConnection () {
         return createConnection;
+    },
+    get fileSize () {
+        return fileSize;
+    },
+    get reclaimSpace () {
+        return reclaimSpace;
     }
 });
+var _promises = require("node:fs/promises");
+var _batchts = require("../batch.js");
+var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
+var _nativets = require("./native.js");
+var _sqlfunctionsts = require("./sql-functions.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -211,6 +225,17 @@ function _async_to_generator(fn) {
 function _class_call_check(instance, Constructor) {
     if (!(instance instanceof Constructor)) throw new TypeError("Cannot call a class as a function");
 }
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
 function _iterable_to_array(iter) {
     if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
         return Array.from(iter);
@@ -219,6 +244,44 @@ function _iterable_to_array(iter) {
 function _non_iterable_spread() {
     throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
 }
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) {
+            symbols = symbols.filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+            });
+        }
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = source != null ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else {
+        ownKeys(Object(source)).forEach(function(key) {
+            Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+        });
+    }
+    return target;
+}
 function _overload_yield(value, kind) {
     this.v = value;
     this.k = kind;
@@ -476,6 +539,168 @@ function checkpointWal(db) {
         });
     })();
 }
+function cacheFilePath(conn) {
+    return _async_to_generator(function() {
+        var _rows_find, rows;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.prepare('PRAGMA database_list')
+                    ];
+                case 1:
+                    return [
+                        4,
+                        _state.sent().all()
+                    ];
+                case 2:
+                    rows = _state.sent();
+                    return [
+                        2,
+                        ((_rows_find = rows.find(function(r) {
+                            return r.name === 'main';
+                        })) === null || _rows_find === void 0 ? void 0 : _rows_find.file) || null
+                    ];
+            }
+        });
+    })();
+}
+function fileSize(path) {
+    return _async_to_generator(function() {
+        var unused;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _state.trys.push([
+                        0,
+                        2,
+                        ,
+                        3
+                    ]);
+                    return [
+                        4,
+                        (0, _promises.stat)(path)
+                    ];
+                case 1:
+                    return [
+                        2,
+                        _state.sent().size
+                    ];
+                case 2:
+                    unused = _state.sent();
+                    return [
+                        2,
+                        null
+                    ];
+                case 3:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function reclaimSpace(path) {
+    return _async_to_generator(function() {
+        var turso, db, size, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _state.trys.push([
+                        0,
+                        12,
+                        ,
+                        13
+                    ]);
+                    return [
+                        4,
+                        (0, _nativets.tursoApi)()
+                    ];
+                case 1:
+                    turso = _state.sent();
+                    return [
+                        4,
+                        turso.connect(path, _object_spread_props(_object_spread({}, _nativets.CONNECT_OPTS), {
+                            experimental: _to_consumable_array(_nativets.CONNECT_OPTS.experimental).concat([
+                                'vacuum'
+                            ])
+                        }))
+                    ];
+                case 2:
+                    db = _state.sent();
+                    _state.label = 3;
+                case 3:
+                    _state.trys.push([
+                        3,
+                        ,
+                        9,
+                        11
+                    ]);
+                    return [
+                        4,
+                        db.exec('VACUUM')
+                    ];
+                case 4:
+                    _state.sent();
+                    return [
+                        4,
+                        fileSize(path)
+                    ];
+                case 5:
+                    size = _state.sent();
+                    if (!(size !== null)) return [
+                        3,
+                        7
+                    ];
+                    return [
+                        4,
+                        (0, _sharedts.setMeta)(createConnection(db), 'compact_size', String(size))
+                    ];
+                case 6:
+                    _state.sent();
+                    _state.label = 7;
+                case 7:
+                    return [
+                        4,
+                        checkpointWal(db)
+                    ];
+                case 8:
+                    _state.sent();
+                    return [
+                        3,
+                        11
+                    ];
+                case 9:
+                    return [
+                        4,
+                        db.close()
+                    ];
+                case 10:
+                    _state.sent();
+                    return [
+                        7
+                    ];
+                case 11:
+                    return [
+                        3,
+                        13
+                    ];
+                case 12:
+                    err = _state.sent();
+                    console.error("sense: turso VACUUM failed, the cache will keep the space until one succeeds: ".concat(err.message));
+                    return [
+                        3,
+                        13
+                    ];
+                case 13:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 function createConnection(db) {
     var conn = {
         exec: function exec(sql) {
@@ -505,7 +730,7 @@ function createConnection(db) {
                             _ = TursoStatementWrapper.bind;
                             return [
                                 4,
-                                db.prepare(sql)
+                                db.prepare((0, _sqlfunctionsts.rewriteFunctions)(sql))
                             ];
                         case 1:
                             return [
@@ -519,8 +744,8 @@ function createConnection(db) {
                 });
             })();
         },
-        runBatch: // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as
-        // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
+        runBatch: // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):
+        // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.
         function runBatch(sql, paramRows) {
             return _async_to_generator(function() {
                 return _ts_generator(this, function(_state) {
@@ -533,13 +758,18 @@ function createConnection(db) {
                                 4,
                                 (0, _transactionts.withTransaction)(conn, function() {
                                     return _async_to_generator(function() {
-                                        var stmt, _stmt, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, err;
+                                        var rewritten, stmt, stmt1, _stmt, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, err;
                                         return _ts_generator(this, function(_state) {
                                             switch(_state.label){
                                                 case 0:
+                                                    rewritten = (0, _batchts.rewriteInsert)(sql, paramRows.length);
+                                                    if (!rewritten) return [
+                                                        3,
+                                                        7
+                                                    ];
                                                     return [
                                                         4,
-                                                        db.prepare(sql)
+                                                        db.prepare(rewritten.sql)
                                                     ];
                                                 case 1:
                                                     stmt = _state.sent();
@@ -548,53 +778,94 @@ function createConnection(db) {
                                                     _state.trys.push([
                                                         2,
                                                         ,
-                                                        11,
-                                                        13
+                                                        4,
+                                                        6
                                                     ]);
-                                                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
-                                                    _state.label = 3;
+                                                    // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack
+                                                    // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).
+                                                    return [
+                                                        4,
+                                                        stmt.run(paramRows.flat())
+                                                    ];
                                                 case 3:
-                                                    _state.trys.push([
+                                                    _state.sent();
+                                                    return [
                                                         3,
-                                                        8,
+                                                        6
+                                                    ];
+                                                case 4:
+                                                    return [
+                                                        4,
+                                                        stmt.close()
+                                                    ];
+                                                case 5:
+                                                    _state.sent();
+                                                    return [
+                                                        7
+                                                    ];
+                                                case 6:
+                                                    return [
+                                                        2
+                                                    ];
+                                                case 7:
+                                                    return [
+                                                        4,
+                                                        db.prepare(sql)
+                                                    ];
+                                                case 8:
+                                                    stmt1 = _state.sent();
+                                                    _state.label = 9;
+                                                case 9:
+                                                    _state.trys.push([
                                                         9,
-                                                        10
+                                                        ,
+                                                        18,
+                                                        20
+                                                    ]);
+                                                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                                                    _state.label = 10;
+                                                case 10:
+                                                    _state.trys.push([
+                                                        10,
+                                                        15,
+                                                        16,
+                                                        17
                                                     ]);
                                                     _iterator = paramRows[Symbol.iterator]();
-                                                    _state.label = 4;
-                                                case 4:
+                                                    _state.label = 11;
+                                                case 11:
                                                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
                                                         3,
-                                                        7
+                                                        14
                                                     ];
                                                     row = _step.value;
                                                     return [
                                                         4,
-                                                        (_stmt = stmt).run.apply(_stmt, _to_consumable_array(row))
+                                                        (_stmt = stmt1).run.apply(_stmt, _to_consumable_array(row))
                                                     ];
-                                                case 5:
+                                                case 12:
                                                     _state.sent();
-                                                    _state.label = 6;
-                                                case 6:
+                                                    _state.label = 13;
+                                                case 13:
                                                     _iteratorNormalCompletion = true;
                                                     return [
                                                         3,
-                                                        4
+                                                        11
                                                     ];
-                                                case 7:
+                                                case 14:
                                                     return [
                                                         3,
-                                                        10
+                                                        17
                                                     ];
-                                                case 8:
+                                                case 15:
                                                     err = _state.sent();
                                                     _didIteratorError = true;
                                                     _iteratorError = err;
                                                     return [
                                                         3,
-                                                        10
+                                                        17
                                                     ];
-                                                case 9:
+                                                case 16:
                                                     try {
                                                         if (!_iteratorNormalCompletion && _iterator.return != null) {
                                                             _iterator.return();
@@ -607,22 +878,22 @@ function createConnection(db) {
                                                     return [
                                                         7
                                                     ];
-                                                case 10:
+                                                case 17:
                                                     return [
                                                         3,
-                                                        13
+                                                        20
                                                     ];
-                                                case 11:
+                                                case 18:
                                                     return [
                                                         4,
-                                                        stmt.close()
+                                                        stmt1.close()
                                                     ];
-                                                case 12:
+                                                case 19:
                                                     _state.sent();
                                                     return [
                                                         7
                                                     ];
-                                                case 13:
+                                                case 20:
                                                     return [
                                                         2
                                                     ];

package/dist/cjs/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as\n    // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = await db.prepare(sql);\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["checkpointWal","createConnection","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","err","exec","console","error","message","conn","sql","prepare","runBatch","paramRows","length","withTransaction","row","close","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QA4CsBA;eAAAA;;QAQNC;eAAAA;;;6BAnD6B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAO7C,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAkCC,SAAeF,cAAca,EAAY;;YAGrCC;;;;;;;;;;oBADP;;wBAAMD,GAAGE,IAAI,CAAC;;;oBAAd;;;;;;oBACOD;oBACPE,QAAQC,KAAK,CAAC,AAAC,2FAAiH,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEnI;;AAEO,SAASjB,iBAAiBY,EAAY;IAC3C,IAAMM,OAAmB;QACjBJ,MAAN,SAAMA,KAAKK,GAAW;;;;;4BACpB;;gCAAMP,GAAGE,IAAI,CAACK;;;4BAAd;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZlB;4BAAsB;;gCAAMW,GAAGQ,OAAO,CAACD;;;4BAAlD;;gCAAO,IAAA,CAAA,EAAA,MAAIlB;;oCAAsB;kCAAqB;;;;YACxD;;QAGMoB,UAFN,+FAA+F;QAC/F,wKAAwK;QACxK,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBN,MACA;;4CACQhB,MAI+BA,OAA9B,2BAAA,mBAAA,gBAAA,WAAA,OAAMuB;;;;oDAJA;;wDAAMb,GAAGQ,OAAO,CAACD;;;oDAAxBjB,OAAO;;;;;;;;;oDAIN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaoB;;;2DAAb,6BAAA,QAAA;;;;oDAAMG,MAAN;oDAAwB;;wDAAMvB,CAAAA,QAAAA,MAAKC,GAAG,OAARD,OAAS,qBAAGuB;;;oDAAlB;;;oDAAxB;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAEL;;wDAAMvB,KAAKwB,KAAK;;;oDAAhB;;;;;;;;;;oCAEJ;mCACAC,0BAAW;;;4BAZb;;;;;;YAcF;;IACF;IACA,OAAOT;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\nimport { rewriteFunctions } from './sql-functions.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(rewriteFunctions(sql)));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and nothing else owns this statement's lifetime.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["cacheFilePath","checkpointWal","createConnection","fileSize","reclaimSpace","TursoStatementWrapper","stmt","run","params","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","db","err","exec","console","error","message","conn","rows","prepare","find","r","name","file","path","stat","size","turso","tursoApi","connect","CONNECT_OPTS","experimental","setMeta","String","close","sql","rewriteFunctions","runBatch","paramRows","length","withTransaction","rewritten","row","rewriteInsert","flat","BEGIN_WRITE"],"mappings":";;;;;;;;;;;QA2DsBA;eAAAA;;QAVAC;eAAAA;;QA2CNC;eAAAA;;QA5BMC;eAAAA;;QAUAC;eAAAA;;;wBA1ED;uBAES;wBACN;6BACqB;wBAEN;8BACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAMjC,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,sCAAN;;aAAMA,sBAGQC,IAAoB;gCAH5BD;QAIF,IAAI,CAACC,IAAI,GAAGA;;iBAJVD;IAOJ,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACC,GAAG,OAAb,YAAc,qBAAGC;;;QAC1B;;IAEA,OAAMC,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGD,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACG,GAAG,OAAb,YAAc,qBAAGD;;;QAC1B;;IAEA,OAAME,GAEL,GAFD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGF,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACrB;;gBAAP;;oBAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACI,GAAG,OAAb,YAAc,qBAAGF;;;QAC1B;;IAEA,OAAOG,OAEN,GAFD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGH,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAC1B;;;;wBAAP;;uCAAA,0CAAO,CAAA,aAAA,IAAI,CAACF,IAAI,EAACK,OAAO,OAAjB,YAAkB,qBAAGH;;;wBAA5B;;;;;;QACF;;IAEAI,OAAAA,OAEC,GAFDA,SAAAA;QACE,OAAO,IAAI,CAACN,IAAI,CAACM,OAAO;IAC1B;IAEAC,OAAAA,cAEC,GAFDA,SAAAA,eAAeC,OAAgB;QAC7B,IAAI,CAACR,IAAI,CAACS,YAAY,CAACD;IACzB;WA7BIT;;AAkCC,SAAeJ,cAAce,EAAY;;YAGrCC;;;;;;;;;;oBADP;;wBAAMD,GAAGE,IAAI,CAAC;;;oBAAd;;;;;;oBACOD;oBACPE,QAAQC,KAAK,CAAC,AAAC,2FAAiH,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEnI;;AAIO,SAAerB,cAAcsB,IAAgB;;YAE3CC,YADDA;;;;oBAAe;;wBAAMD,KAAKE,OAAO,CAAC;;;oBAA1B;;wBAAO,cAA4Cd,GAAG;;;oBAA9Da,OAAQ;oBACd;;wBAAOA,EAAAA,aAAAA,KAAKE,IAAI,CAAC,SAACC;mCAAMA,EAAEC,IAAI,KAAK;wCAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;;;;IACtD;;AAEO,SAAezB,SAAS0B,IAAY;;;;;;;;;;;;oBAE/B;;wBAAMC,IAAAA,cAAI,EAACD;;;oBAAnB;;wBAAQ,cAAkBE,IAAI;;;;oBAE9B;;wBAAO;;;;;;;;IAEX;;AAIO,SAAe3B,aAAayB,IAAY;;YAErCG,OACAhB,IAGEe,MAODd;;;;;;;;;;oBAXO;;wBAAMgB,IAAAA,kBAAQ;;;oBAAtBD,QAAQ;oBACH;;wBAAMA,MAAME,OAAO,CAACL,MAAM,wCAAKM,sBAAY;4BAAEC,cAAc,AAAC,qBAAGD,sBAAY,CAACC,YAAY;gCAAE;;;;;oBAA/FpB,KAAK;;;;;;;;;oBAET;;wBAAMA,GAAGE,IAAI,CAAC;;;oBAAd;oBACa;;wBAAMf,SAAS0B;;;oBAAtBE,OAAO;yBAETA,CAAAA,SAAS,IAAG,GAAZA;;;;oBAAe;;wBAAMM,IAAAA,iBAAO,EAACnC,iBAAiBc,KAAK,gBAAgBsB,OAAOP;;;oBAA3D;;;oBACnB;;wBAAM9B,cAAce;;;oBAApB;;;;;;oBAEA;;wBAAMA,GAAGuB,KAAK;;;oBAAd;;;;;;;;;;oBAEKtB;oBACPE,QAAQC,KAAK,CAAC,AAAC,iFAAuG,OAAvB,AAACH,IAAcI,OAAO;;;;;;;;;;;IAEzH;;AAEO,SAASnB,iBAAiBc,EAAY;IAC3C,IAAMM,OAAmB;QACjBJ,MAAN,SAAMA,KAAKsB,GAAW;;;;;4BACpB;;gCAAMxB,GAAGE,IAAI,CAACsB;;;4BAAd;;;;;;YACF;;QACMhB,SAAN,SAAMA,QAAQgB,GAAW;;;;;;gCACZnC;4BAAsB;;gCAAMW,GAAGQ,OAAO,CAACiB,IAAAA,gCAAgB,EAACD;;;4BAAnE;;gCAAO,IAAA,CAAA,EAAA,MAAInC;;oCAAsB;kCAAuC;;;;YAC1E;;QAGMqC,UAFN,gGAAgG;QAChG,wGAAwG;QACxG,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAMC,IAAAA,8BAAe,EACnBvB,MACA;;4CACQwB,WAEExC,MAYFA,OAE+BA,OAA9B,2BAAA,mBAAA,gBAAA,WAAA,OAAMyC;;;;oDAhBPD,YAAYE,IAAAA,sBAAa,EAACR,KAAKG,UAAUC,MAAM;yDACjDE,WAAAA;;;;oDACW;;wDAAM9B,GAAGQ,OAAO,CAACsB,UAAUN,GAAG;;;oDAArClC,OAAO;;;;;;;;;oDAEX,uFAAuF;oDACvF,uFAAuF;oDACvF;;wDAAMA,KAAKC,GAAG,CAACoC,UAAUM,IAAI;;;oDAA7B;;;;;;oDAEA;;wDAAM3C,KAAKiC,KAAK;;;oDAAhB;;;;;oDAEF;;;;oDAIW;;wDAAMvB,GAAGQ,OAAO,CAACgB;;;oDAAxBlC,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaqC;;;2DAAb,6BAAA,QAAA;;;;oDAAMI,MAAN;oDAAwB;;wDAAMzC,CAAAA,QAAAA,OAAKC,GAAG,OAARD,OAAS,qBAAGyC;;;oDAAlB;;;oDAAxB;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAEL;;wDAAMzC,MAAKiC,KAAK;;;oDAAhB;;;;;;;;;;oCAEJ;mCACAW,0BAAW;;;4BAxBb;;;;;;YA0BF;;IACF;IACA,OAAO5B;AACT"}

package/dist/cjs/store/turso/lexical-text.d.cts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/cjs/store/turso/lexical-text.d.ts

@@ -0,0 +1 @@
+export declare function stemFolded(text: string): string;

package/dist/cjs/store/turso/lexical-text.js

@@ -0,0 +1,40 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "stemFolded", {
+    enumerable: true,
+    get: function() {
+        return stemFolded;
+    }
+});
+var _stemmer = require("stemmer");
+var _segmentts = require("../../text/segment.js");
+function stemFolded(text) {
+    var out = '';
+    var end = 0;
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = (0, _segmentts.searchTokens)(text)[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var token = _step.value;
+            out += text.slice(end, token.start).replace(RegExp("[\\u0027\\u2019]", "gu"), ' ');
+            out += token.unspaced ? token.text : (0, _stemmer.stemmer)((0, _segmentts.foldForSearch)(token.text));
+            end = token.end;
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return out + text.slice(end).replace(RegExp("[\\u0027\\u2019]", "gu"), ' ');
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/turso/lexical-text.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/lexical-text.ts"],"sourcesContent":["import { stemmer } from 'stemmer';\nimport { foldForSearch, searchTokens } from '../../text/segment.ts';\n\nexport function stemFolded(text: string): string {\n  let out = '';\n  let end = 0;\n  for (const token of searchTokens(text)) {\n    out += text.slice(end, token.start).replace(/[\\u0027\\u2019]/gu, ' ');\n    out += token.unspaced ? token.text : stemmer(foldForSearch(token.text));\n    end = token.end;\n  }\n  return out + text.slice(end).replace(/[\\u0027\\u2019]/gu, ' ');\n}\n"],"names":["stemFolded","text","out","end","searchTokens","token","slice","start","replace","unspaced","stemmer","foldForSearch"],"mappings":";;;;+BAGgBA;;;eAAAA;;;uBAHQ;yBACoB;AAErC,SAASA,WAAWC,IAAY;IACrC,IAAIC,MAAM;IACV,IAAIC,MAAM;QACL,kCAAA,2BAAA;;QAAL,QAAK,YAAeC,IAAAA,uBAAY,EAACH,0BAA5B,SAAA,6BAAA,QAAA,yBAAA,iCAAmC;YAAnC,IAAMI,QAAN;YACHH,OAAOD,KAAKK,KAAK,CAACH,KAAKE,MAAME,KAAK,EAAEC,OAAO,CAAC,kCAAoB;YAChEN,OAAOG,MAAMI,QAAQ,GAAGJ,MAAMJ,IAAI,GAAGS,IAAAA,gBAAO,EAACC,IAAAA,wBAAa,EAACN,MAAMJ,IAAI;YACrEE,MAAME,MAAMF,GAAG;QACjB;;QAJK;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IAKL,OAAOD,MAAMD,KAAKK,KAAK,CAACH,KAAKK,OAAO,CAAC,kCAAoB;AAC3D"}