sensemaking 0.20.0 → 0.21.0

package/dist/cjs/store/sqlite/reconcile.d.ts

@@ -1,5 +1,5 @@
 import type { ParsedDoc } from '../../scan/index.js';
 import type { ReconcileDialect } from '../types.js';
 export declare const INSERT_CONTENT_SQL = "INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)";
-export declare function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[];
+export declare function contentRow(doc: ParsedDoc): unknown[];
 export declare const sqliteDialect: ReconcileDialect;

package/dist/cjs/store/sqlite/reconcile.js

@@ -19,7 +19,6 @@ _export(exports, {
         return sqliteDialect;
     }
 });
-var _indexts = require("../../config/index.js");
 var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
@@ -154,21 +153,20 @@ function _ts_generator(thisArg, body) {
 // SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).
 var MAX_FRONTMATTER_COLUMNS = 2000;
 var INSERT_CONTENT_SQL = 'INSERT INTO content (rowid, title, summary, text, "path", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE "path" = ?), ?, ?, ?, ?, ?, ?, ?)';
-function contentRow(doc, segmenting) {
+function contentRow(doc) {
     return [
         doc.relPath,
         doc.search.title,
         doc.search.summary,
         doc.search.text,
         doc.relPath,
-        segmenting ? (0, _segmentts.segmentField)(doc.search.title) : '',
-        segmenting ? (0, _segmentts.segmentField)(doc.search.summary) : '',
-        segmenting ? (0, _segmentts.segmentField)(doc.search.text) : ''
+        (0, _segmentts.segmentField)(doc.search.title),
+        (0, _segmentts.segmentField)(doc.search.summary),
+        (0, _segmentts.segmentField)(doc.search.text)
     ];
 }
-function reconcileContent(conn, touched, docs, _delta, cfg) {
+function reconcileContent(conn, touched, docs, _delta, _cfg) {
     return _async_to_generator(function() {
-        var segmenting;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
@@ -192,19 +190,85 @@ function reconcileContent(conn, touched, docs, _delta, cfg) {
                         3,
                         4
                     ];
-                    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over
-                    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.
-                    segmenting = (0, _indexts.contentTokenize)(cfg) === undefined;
                     return [
                         4,
-                        conn.runBatch(INSERT_CONTENT_SQL, docs.map(function(doc) {
-                            return contentRow(doc, segmenting);
-                        }))
+                        conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow))
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+// SQLite's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.
+function addColumns(conn, names) {
+    return _async_to_generator(function() {
+        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, name, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        6,
+                        7,
+                        8
+                    ]);
+                    _iterator = names[Symbol.iterator]();
+                    _state.label = 2;
+                case 2:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        5
+                    ];
+                    name = _step.value;
+                    return [
+                        4,
+                        conn.exec("ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name)))
                     ];
                 case 3:
                     _state.sent();
                     _state.label = 4;
                 case 4:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        2
+                    ];
+                case 5:
+                    return [
+                        3,
+                        8
+                    ];
+                case 6:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 8:
                     return [
                         2
                     ];
@@ -223,9 +287,7 @@ var sqliteDialect = {
             throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ").concat(MAX_FRONTMATTER_COLUMNS, "; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
         }
     },
-    columnTypeSuffix: function columnTypeSuffix() {
-        return '';
-    },
+    addColumns: addColumns,
     reconcileContent: reconcileContent,
     recordDuration: _sharedts.recordReconcileDuration
 };

package/dist/cjs/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) {\n    // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n    // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n    const segmenting = contentTokenize(cfg) === undefined;\n    await conn.runBatch(\n      INSERT_CONTENT_SQL,\n      docs.map((doc) => contentRow(doc, segmenting))\n    );\n  }\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["INSERT_CONTENT_SQL","contentRow","sqliteDialect","MAX_FRONTMATTER_COLUMNS","doc","segmenting","relPath","search","title","summary","text","segmentField","reconcileContent","conn","touched","docs","_delta","cfg","length","runBatch","map","p","contentTokenize","undefined","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAkBaA;eAAAA;;QAIGC;eAAAA;;QAwBHC;eAAAA;;;uBA7CmB;wBACL;yBAGE;wBACW;6BACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,IAAMC,0BAA0B;AAIzB,IAAMH,qBAAqB;AAI3B,SAASC,WAAWG,GAAc,EAAEC,UAAmB;IAC5D,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaM,IAAAA,uBAAY,EAACP,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,SAAeE,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,GAAW;;YAYjHZ;;;;yBATJS,CAAAA,QAAQI,MAAM,GAAG,CAAA,GAAjBJ;;;;oBACF;;wBAAMD,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAKEN,CAAAA,KAAKG,MAAM,GAAG,CAAA,GAAdH;;;;oBACF,wFAAwF;oBACxF,+EAA+E;oBACzEV,aAAaiB,IAAAA,wBAAe,EAACL,SAASM;oBAC5C;;wBAAMV,KAAKM,QAAQ,CACjBnB,oBACAe,KAAKK,GAAG,CAAC,SAAChB;mCAAQH,WAAWG,KAAKC;;;;oBAFpC;;;;;;;;IAKJ;;AAEO,IAAMH,gBAAkC;IAC7CsB,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQxB,yBAAyB;YACnC,MAAM,IAAIyB,oBAAU,CAClB,gBACA,AAAC,0BAA2GzB,OAAlFwB,OAAM,8EAAoG,OAAxBxB,yBAAwB;QAExI;IACF;IACA0B,kBAAkB,SAAlBA;eAAwB;;IACxBjB,kBAAAA;IACAkB,gBAAgBC,iCAAuB;AACzC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is FTS5-virtual with \"_seg\" sidecars for lexical.ts.\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcileContent() and the tokenize-only rebuild in open.ts, so both prepare the\n// same literal instead of two copies drifting apart.\nexport const INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nexport function contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmentField(doc.search.title), segmentField(doc.search.summary), segmentField(doc.search.text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta, _cfg: Config): Promise<void> {\n  // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n  // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n  // `touched` already covers a path a concurrent reconcile turned out to have created first --\n  // see reconcile.ts's post-lock revalidation, which folds such paths in before calling here.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n      touched.map((p) => [p])\n    );\n\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\n// SQLite's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const sqliteDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    // Fence before ALTERing: SQLite's own failure past this point is a raw\n    // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["INSERT_CONTENT_SQL","contentRow","sqliteDialect","MAX_FRONTMATTER_COLUMNS","doc","relPath","search","title","summary","text","segmentField","reconcileContent","conn","touched","docs","_delta","_cfg","length","runBatch","map","p","addColumns","names","name","exec","quoteIdent","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAiBaA;eAAAA;;QAIGC;eAAAA;;QAuBHC;eAAAA;;;wBA3Cc;yBAGE;wBACuB;6BACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG5B,qFAAqF;AACrF,mFAAmF;AAEnF,8FAA8F;AAC9F,IAAMC,0BAA0B;AAIzB,IAAMH,qBAAqB;AAI3B,SAASC,WAAWG,GAAc;IACvC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;QAAEL,IAAIC,OAAO;QAAEK,IAAAA,uBAAY,EAACN,IAAIE,MAAM,CAACC,KAAK;QAAGG,IAAAA,uBAAY,EAACN,IAAIE,MAAM,CAACE,OAAO;QAAGE,IAAAA,uBAAY,EAACN,IAAIE,MAAM,CAACG,IAAI;KAAE;AAC3L;AAEA,SAAeE,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB,EAAEC,IAAY;;;;;yBAKtHH,CAAAA,QAAQI,MAAM,GAAG,CAAA,GAAjBJ;;;;oBACF;;wBAAMD,KAAKM,QAAQ,CACjB,sFACAL,QAAQM,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAKEN,CAAAA,KAAKG,MAAM,GAAG,CAAA,GAAdH;;;;oBAAiB;;wBAAMF,KAAKM,QAAQ,CAAClB,oBAAoBc,KAAKK,GAAG,CAAClB;;;oBAAjD;;;;;;;;IACvB;;AAEA,0FAA0F;AAC1F,SAAeoB,WAAWT,IAAgB,EAAEU,KAAe;;YACpD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcD;;;2BAAd,6BAAA,QAAA;;;;oBAAMC,OAAN;oBAAqB;;wBAAMX,KAAKY,IAAI,CAAC,AAAC,sCAAsD,OAAjBC,IAAAA,oBAAU,EAACF;;;oBAAjE;;;oBAArB;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEO,IAAMrB,gBAAkC;IAC7CwB,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,uEAAuE;QACvE,sGAAsG;QACtG,IAAIA,QAAQ1B,yBAAyB;YACnC,MAAM,IAAI2B,oBAAU,CAClB,gBACA,AAAC,0BAA2G3B,OAAlF0B,OAAM,8EAAoG,OAAxB1B,yBAAwB;QAExI;IACF;IACAkB,YAAAA;IACAV,kBAAAA;IACAoB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/sqlite/sql-functions.d.cts

@@ -1,2 +1,2 @@
 import type { DatabaseSync } from 'node:sqlite';
-export declare function registerFunctions(db: DatabaseSync, segmenting: boolean): void;
+export declare function registerFunctions(db: DatabaseSync): void;

package/dist/cjs/store/sqlite/sql-functions.d.ts

@@ -1,2 +1,2 @@
 import type { DatabaseSync } from 'node:sqlite';
-export declare function registerFunctions(db: DatabaseSync, segmenting: boolean): void;
+export declare function registerFunctions(db: DatabaseSync): void;

package/dist/cjs/store/sqlite/sql-functions.js

@@ -10,7 +10,7 @@ Object.defineProperty(exports, "registerFunctions", {
 });
 var _segmentts = require("../../text/segment.js");
 var _sqlfunctionsts = require("../sql-functions.js");
-function registerFunctions(db, segmenting) {
+function registerFunctions(db) {
     db.function('has', {
         deterministic: true,
         varargs: false
@@ -20,13 +20,13 @@ function registerFunctions(db, segmenting) {
         varargs: true
     }, _sqlfunctionsts.basenameImpl);
     // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back and matches
-    // nothing, so the same transform is available to hand-written SQL by name. `segmenting` is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) === undefined), passed in by open().
+    // nothing, so the same transform is available to hand-written SQL by name.
     db.function('segment', {
         deterministic: true,
         varargs: false
     }, function(terms) {
         var text = terms === null || terms === undefined ? '' : String(terms);
-        return segmenting ? (0, _segmentts.segmentMatch)(text) : text;
+        return (0, _segmentts.segmentMatch)(text);
     });
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/sql-functions.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\n\nexport function registerFunctions(db: DatabaseSync, segmenting: boolean): void {\n  db.function('has', { deterministic: true, varargs: false }, hasImpl);\n\n  db.function('basename', { deterministic: true, varargs: true }, basenameImpl);\n\n  // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back and matches\n  // nothing, so the same transform is available to hand-written SQL by name. `segmenting` is the same config predicate commands.ts's search() gates on (contentTokenize(cfg) === undefined), passed in by open().\n  db.function('segment', { deterministic: true, varargs: false }, (terms: unknown): string => {\n    const text = terms === null || terms === undefined ? '' : String(terms);\n    return segmenting ? segmentMatch(text) : text;\n  });\n}\n"],"names":["registerFunctions","db","segmenting","function","deterministic","varargs","hasImpl","basenameImpl","terms","text","undefined","String","segmentMatch"],"mappings":";;;;+BAIgBA;;;eAAAA;;;yBAHa;8BACS;AAE/B,SAASA,kBAAkBC,EAAgB,EAAEC,UAAmB;IACrED,GAAGE,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAGC,uBAAO;IAEnEL,GAAGE,QAAQ,CAAC,YAAY;QAAEC,eAAe;QAAMC,SAAS;IAAK,GAAGE,4BAAY;IAE5E,iGAAiG;IACjG,gNAAgN;IAChNN,GAAGE,QAAQ,CAAC,WAAW;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACG;QAC/D,IAAMC,OAAOD,UAAU,QAAQA,UAAUE,YAAY,KAAKC,OAAOH;QACjE,OAAON,aAAaU,IAAAA,uBAAY,EAACH,QAAQA;IAC3C;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/sql-functions.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\n\nexport function registerFunctions(db: DatabaseSync): void {\n  db.function('has', { deterministic: true, varargs: false }, hasImpl);\n\n  db.function('basename', { deterministic: true, varargs: true }, basenameImpl);\n\n  // Raw `content MATCH '<unspaced text>'` cannot be rewritten behind the author's back and matches\n  // nothing, so the same transform is available to hand-written SQL by name.\n  db.function('segment', { deterministic: true, varargs: false }, (terms: unknown): string => {\n    const text = terms === null || terms === undefined ? '' : String(terms);\n    return segmentMatch(text);\n  });\n}\n"],"names":["registerFunctions","db","function","deterministic","varargs","hasImpl","basenameImpl","terms","text","undefined","String","segmentMatch"],"mappings":";;;;+BAIgBA;;;eAAAA;;;yBAHa;8BACS;AAE/B,SAASA,kBAAkBC,EAAgB;IAChDA,GAAGC,QAAQ,CAAC,OAAO;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAGC,uBAAO;IAEnEJ,GAAGC,QAAQ,CAAC,YAAY;QAAEC,eAAe;QAAMC,SAAS;IAAK,GAAGE,4BAAY;IAE5E,iGAAiG;IACjG,2EAA2E;IAC3EL,GAAGC,QAAQ,CAAC,WAAW;QAAEC,eAAe;QAAMC,SAAS;IAAM,GAAG,SAACG;QAC/D,IAAMC,OAAOD,UAAU,QAAQA,UAAUE,YAAY,KAAKC,OAAOH;QACjE,OAAOI,IAAAA,uBAAY,EAACH;IACtB;AACF"}

package/dist/cjs/store/sqlite/store.d.cts

@@ -1,5 +1,4 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection): Store;

package/dist/cjs/store/sqlite/store.d.ts

@@ -1,5 +1,4 @@
 import type { DatabaseSync } from 'node:sqlite';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store;
+export declare function createStore(db: DatabaseSync, conn: Connection): Store;

package/dist/cjs/store/sqlite/store.js

@@ -16,7 +16,6 @@ _export(exports, {
         return createStore;
     }
 });
-var _indexts = require("../../config/index.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
@@ -369,7 +368,7 @@ var CAPABILITIES = new Set([
     'vectors',
     'sql-functions'
 ]);
-function createStore(db, conn, cfg) {
+function createStore(db, conn) {
     return {
         name: 'sqlite',
         capabilities: CAPABILITIES,
@@ -467,7 +466,7 @@ function createStore(db, conn, cfg) {
                     return _ts_generator(this, function(_state) {
                         return [
                             2,
-                            (0, _lexicalts.queryLexical)(conn, terms, opts, (0, _indexts.contentTokenize)(cfg) === undefined)
+                            (0, _lexicalts.queryLexical)(conn, terms, opts)
                         ];
                     });
                 })();

package/dist/cjs/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAWaA;eAAAA;;QAIGC;eAAAA;;;uBAbgB;wBACL;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB;AAIzI,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW;IACzE,OAAO;QACLC,MAAM;QACNC,cAAcP;QACRQ,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAML,KAAKI,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOL,KAAKM,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMR,KAAKO,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACX,MAAMU;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACd;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMe,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACf,MAAMa,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACrB,MAAMmB,OAAOC,MAAME,IAAAA,wBAAe,EAACrB,SAASsB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC1B;;;gBACrB;;YACM2B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC7B,MAAM4B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACnC,MAAM+B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACtC,MAAMqC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACxC,MAAMqC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM3C,GAAGO,OAAO,CAAC,uBAAuBqC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGxC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB0C;;wBAAAA,OAAOhD,GAAGO,OAAO,CAACD;wBACxB0C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJpD,GAAGoD,KAAK;;;;;YACV;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with the builder's own reconcile call so transaction depth is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QASaA;eAAAA;;QAIGC;eAAAA;;;wBAZW;6BACK;yBAEU;4BACf;yBACE;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB;AAIzI,SAASD,YAAYE,EAAgB,EAAEC,IAAgB;IAC5D,OAAO;QACLC,MAAM;QACNC,cAAcN;QACRO,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMJ,KAAKG,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOJ,KAAKK,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMP,KAAKM,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACV,MAAMS;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACb;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMc,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACd,MAAMY,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACpB,MAAMkB,OAAOC;;;gBACnC;;QACF;QACAE,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAACvB;;;gBACrB;;YACMwB,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC1B,MAAMyB;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAAChC,MAAM4B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEf,IAAI;;;wBACtB;;4BAAOgB,IAAAA,uBAAW,EAACnC,MAAMkC,MAAMf;;;gBACjC;;YACMiB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACrC,MAAMkC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAMxC,GAAGM,OAAO,CAAC,uBAAuBmC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGtC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBwC;;wBAAAA,OAAO7C,GAAGM,OAAO,CAACD;wBACxBwC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLjC,SAAS,SAATA;2CAAegC,KAAKhC,OAAO;;gCAC3B,2EAA2E;gCAC3EkC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJjD,GAAGiD,KAAK;;;;;YACV;;IACF;AACF"}

package/dist/cjs/store/turso/open.js

@@ -474,6 +474,12 @@ var tursoOpenDialect = {
     reconcileDialect: _reconcilets.tursoDialect,
     connect: connect,
     close: close,
+    // "Locking error: Failed locking file ...", worded per platform: posix "File is locked by another
+    // process", Windows "another process has locked a portion of the file (os error 33)". Distinct from
+    // the write-time "database is locked" its connect-time `timeout` covers; that one never reaches here.
+    isLocked: function isLocked(err) {
+        return /File is locked by another process|locked a portion of the file/.test(err.message);
+    },
     ensureSchema: ensureSchema,
     setDerivedBusyTimeout: setDerivedBusyTimeout,
     createStore: function createStore(handle, conn) {

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAmESC;eAAAA;;QAXTC;eAAAA;;;uBAxEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACO;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeE,sBAAsBjB,OAAiB,EAAEC,IAAgB,EAAEiB,EAAU;;;;;oBAClF;;wBAAMjB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHa;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEZ,QAAQQ;4BAAIvB,MAAM8B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM1B,mBAA0C;IACrDkC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAJ,OAAAA;IACAhB,cAAAA;IACAkB,uBAAAA;IACAmB,aAAa,SAAbA,YAAcpB,QAAQf;eAASmC,IAAAA,oBAAW,EAACpB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOmC,IAAAA,uBAAe,EAACnC,KAAKJ;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAuESC;eAAAA;;QAfTC;eAAAA;;;uBAxEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACO;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeE,sBAAsBjB,OAAiB,EAAEC,IAAgB,EAAEiB,EAAU;;;;;oBAClF;;wBAAMjB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHa;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEZ,QAAQQ;4BAAIvB,MAAM8B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM1B,mBAA0C;IACrDkC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAJ,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGqB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG7B,cAAAA;IACAkB,uBAAAA;IACAqB,aAAa,SAAbA,YAActB,QAAQf;eAASqC,IAAAA,oBAAW,EAACtB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKJ;;;IAC9B"}

package/dist/cjs/store/turso/reconcile.js

@@ -349,6 +349,77 @@ function reconcileContent(conn, touched, docs, delta) {
         });
     })();
 }
+// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.
+function addColumns(conn, names) {
+    return _async_to_generator(function() {
+        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, name, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        6,
+                        7,
+                        8
+                    ]);
+                    _iterator = names[Symbol.iterator]();
+                    _state.label = 2;
+                case 2:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        5
+                    ];
+                    name = _step.value;
+                    return [
+                        4,
+                        conn.exec("ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name)))
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        2
+                    ];
+                case 5:
+                    return [
+                        3,
+                        8
+                    ];
+                case 6:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 8:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 var tursoDialect = {
     beginMode: function beginMode() {
         return _transactionts.BEGIN_WRITE;
@@ -358,9 +429,7 @@ var tursoDialect = {
             throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing turso's SELECT result-set column limit (").concat(MAX_FRONTMATTER_COLUMNS, "; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
         }
     },
-    columnTypeSuffix: function columnTypeSuffix() {
-        return '';
-    },
+    addColumns: addColumns,
     reconcileContent: reconcileContent,
     recordDuration: _sharedts.recordReconcileDuration
 };

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","exec","runBatch","map","p","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAcAC;eAAAA;;QA6BAC;eAAAA;;;wBA9Dc;yBAGI;wBACS;6BACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUrB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,6FAA6F;AAC7F,gGAAgG;AAChG,SAASG,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcpB;;;2BAAd,6BAAA,QAAA;;;;oBAAMqB,OAAN;oBAAiC;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,wBAA4B,OAALN;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKa,QAAQ,CACjB,wCACAZ,QAAQa,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEb,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKa,QAAQ,CAACrB,oBAAoBU,KAAKY,GAAG,CAACrB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAarB;;;2BAAb,8BAAA,SAAA;;;;oBAAMuB,MAAN;oBAA8B;;wBAAMP,KAAKY,IAAI,CAACL;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEO,IAAMpB,eAAiC;IAC5C6B,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ5B,yBAAyB;YACnC,MAAM,IAAI6B,oBAAU,CAClB,gBACA,AAAC,0BAA4F7B,OAAnE4B,OAAM,+DAAqF,OAAxB5B,yBAAwB;QAEzH;IACF;IACA8B,kBAAkB,SAAlBA;eAAwB;;IACxBtB,kBAAAA;IACAuB,gBAAgBC,iCAAuB;AACzC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","exec","runBatch","map","p","addColumns","names","quoteIdent","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAcAC;eAAAA;;QAkCAC;eAAAA;;;wBAnEc;yBAGI;wBACqB;6BACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUrB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,6FAA6F;AAC7F,gGAAgG;AAChG,SAASG,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcpB;;;2BAAd,6BAAA,QAAA;;;;oBAAMqB,OAAN;oBAAiC;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,wBAA4B,OAALN;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKa,QAAQ,CACjB,wCACAZ,QAAQa,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEb,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKa,QAAQ,CAACrB,oBAAoBU,KAAKY,GAAG,CAACrB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAarB;;;2BAAb,8BAAA,SAAA;;;;oBAAMuB,MAAN;oBAA8B;;wBAAMP,KAAKY,IAAI,CAACL;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEA,yFAAyF;AACzF,SAAeS,WAAWhB,IAAgB,EAAEiB,KAAe;;YACpD,2BAAA,mBAAA,gBAAA,WAAA,OAAMX;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcW;;;2BAAd,6BAAA,QAAA;;;;oBAAMX,OAAN;oBAAqB;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,sCAAsD,OAAjBM,IAAAA,oBAAU,EAACZ;;;oBAAjE;;;oBAArB;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEO,IAAMnB,eAAiC;IAC5CgC,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIgC,oBAAU,CAClB,gBACA,AAAC,0BAA4FhC,OAAnE+B,OAAM,+DAAqF,OAAxB/B,yBAAwB;QAEzH;IACF;IACAyB,YAAAA;IACAjB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/types.d.cts

@@ -25,7 +25,7 @@ export interface Connection {
 export interface ReconcileDialect {
     beginMode(): string;
     checkColumnLimit(count: number): void;
-    columnTypeSuffix(): string;
+    addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
 }
@@ -38,9 +38,7 @@ export interface OpenDialect<Handle> {
         conn: Connection;
     }>;
     close(handle: Handle): Promise<void>;
-    partialRebuild?(handle: Handle, conn: Connection, cfg: Config, changedKeys: Set<string>): Promise<boolean>;
-    postSchemaRebuild?(handle: Handle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]>;
-    extraRebuildReason?(handle: Handle): string | null;
+    isLocked?(err: Error): boolean;
     ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
     setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;
     createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;

package/dist/cjs/store/types.d.ts

@@ -25,7 +25,7 @@ export interface Connection {
 export interface ReconcileDialect {
     beginMode(): string;
     checkColumnLimit(count: number): void;
-    columnTypeSuffix(): string;
+    addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
 }
@@ -38,9 +38,7 @@ export interface OpenDialect<Handle> {
         conn: Connection;
     }>;
     close(handle: Handle): Promise<void>;
-    partialRebuild?(handle: Handle, conn: Connection, cfg: Config, changedKeys: Set<string>): Promise<boolean>;
-    postSchemaRebuild?(handle: Handle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]>;
-    extraRebuildReason?(handle: Handle): string | null;
+    isLocked?(err: Error): boolean;
     ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
     setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;
     createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Type suffix for ALTER TABLE ADD COLUMN: '' for sqlite/turso, ' VARIANT' for duckdb.\n  columnTypeSuffix(): string;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db, tokenize}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // A signature change fully explained by this dialect's own carve-out (sqlite's tokenize-only\n  // partial rebuild): performs the DDL swap and returns true, or false when this change isn't one.\n  // Absent for duckdb/turso, which have no such carve-out.\n  partialRebuild?(handle: Handle, conn: Connection, cfg: Config, changedKeys: Set<string>): Promise<boolean>;\n  // Runs after ensureSchema when partialRebuild returned true this attempt: repopulates whatever\n  // the DDL swap emptied and records the adopted signature. sqlite-only.\n  postSchemaRebuild?(handle: Handle, conn: Connection, cfg: Config, baseDir: string, wantFeatures: string): Promise<string[]>;\n  // A rebuild trigger independent of the schema/feature signature (sqlite: the content table's own\n  // tokenizer no longer matches meta). Returns the rebuild reason, or null when none applies.\n  extraRebuildReason?(handle: Handle): string | null;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB"}

package/dist/esm/config/access.d.ts

@@ -21,5 +21,4 @@ export declare function embedConfig(cfg: Config): {
     languages?: string[];
     chunkTokens?: number;
 } | null;
-export declare function contentTokenize(cfg: Config): string | undefined;
 export declare function featureSignature(cfg: Config, features: Feature[]): string;