sensemaking 0.18.2 → 0.18.3

package/dist/cjs/store/sqlite/fieldStats.js

@@ -0,0 +1,201 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "fieldStats", {
+    enumerable: true,
+    get: function() {
+        return fieldStats;
+    }
+});
+var _sharedts = require("../shared.js");
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
+    try {
+        var info = gen[key](arg);
+        var value = info.value;
+    } catch (error) {
+        reject(error);
+        return;
+    }
+    if (info.done) resolve(value);
+    else Promise.resolve(value).then(_next, _throw);
+}
+function _async_to_generator(fn) {
+    return function() {
+        var self = this, args = arguments;
+        return new Promise(function(resolve, reject) {
+            var gen = fn.apply(self, args);
+            function _next(value) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "next", value);
+            }
+            function _throw(err) {
+                asyncGeneratorStep(gen, resolve, reject, _next, _throw, "throw", err);
+            }
+            _next(undefined);
+        });
+    };
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _ts_generator(thisArg, body) {
+    var f, y, t, _ = {
+        label: 0,
+        sent: function() {
+            if (t[0] & 1) throw t[1];
+            return t[1];
+        },
+        trys: [],
+        ops: []
+    }, g = Object.create((typeof Iterator === "function" ? Iterator : Object).prototype), d = Object.defineProperty;
+    return d(g, "next", {
+        value: verb(0)
+    }), d(g, "throw", {
+        value: verb(1)
+    }), d(g, "return", {
+        value: verb(2)
+    }), typeof Symbol === "function" && d(g, Symbol.iterator, {
+        value: function() {
+            return this;
+        }
+    }), g;
+    function verb(n) {
+        return function(v) {
+            return step([
+                n,
+                v
+            ]);
+        };
+    }
+    function step(op) {
+        if (f) throw new TypeError("Generator is already executing.");
+        while(g && (g = 0, op[0] && (_ = 0)), _)try {
+            if (f = 1, y && (t = op[0] & 2 ? y["return"] : op[0] ? y["throw"] || ((t = y["return"]) && t.call(y), 0) : y.next) && !(t = t.call(y, op[1])).done) return t;
+            if (y = 0, t) op = [
+                op[0] & 2,
+                t.value
+            ];
+            switch(op[0]){
+                case 0:
+                case 1:
+                    t = op;
+                    break;
+                case 4:
+                    _.label++;
+                    return {
+                        value: op[1],
+                        done: false
+                    };
+                case 5:
+                    _.label++;
+                    y = op[1];
+                    op = [
+                        0
+                    ];
+                    continue;
+                case 7:
+                    op = _.ops.pop();
+                    _.trys.pop();
+                    continue;
+                default:
+                    if (!(t = _.trys, t = t.length > 0 && t[t.length - 1]) && (op[0] === 6 || op[0] === 2)) {
+                        _ = 0;
+                        continue;
+                    }
+                    if (op[0] === 3 && (!t || op[1] > t[0] && op[1] < t[3])) {
+                        _.label = op[1];
+                        break;
+                    }
+                    if (op[0] === 6 && _.label < t[1]) {
+                        _.label = t[1];
+                        t = op;
+                        break;
+                    }
+                    if (t && _.label < t[2]) {
+                        _.label = t[2];
+                        _.ops.push(op);
+                        break;
+                    }
+                    if (t[2]) _.ops.pop();
+                    _.trys.pop();
+                    continue;
+            }
+            op = body.call(thisArg, _);
+        } catch (e) {
+            op = [
+                6,
+                e
+            ];
+            y = 0;
+        } finally{
+            f = t = 0;
+        }
+        if (op[0] & 5) throw op[1];
+        return {
+            value: op[0] ? op[1] : void 0,
+            done: true
+        };
+    }
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+function fieldStats(conn, columns, scopeWhere) {
+    return _async_to_generator(function() {
+        var exprs, stmt, row;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    exprs = columns.map(function(name, i) {
+                        var quoted = (0, _sharedts.quoteIdent)(name);
+                        return "COUNT(".concat(quoted, ") AS n").concat(i, ", GROUP_CONCAT(DISTINCT typeof(").concat(quoted, ")) FILTER (WHERE ").concat(quoted, " IS NOT NULL) AS t").concat(i);
+                    });
+                    return [
+                        4,
+                        conn.prepare("SELECT ".concat(exprs.join(', '), " FROM frontmatter ").concat(scopeWhere))
+                    ];
+                case 1:
+                    stmt = _state.sent();
+                    return [
+                        4,
+                        stmt.get()
+                    ];
+                case 2:
+                    row = _state.sent();
+                    return [
+                        2,
+                        columns.map(function(name, i) {
+                            var _row_, _row_1;
+                            return {
+                                field: name,
+                                coverage: Number((_row_ = row["n".concat(i)]) !== null && _row_ !== void 0 ? _row_ : 0),
+                                type: _to_consumable_array(new Set(((_row_1 = row["t".concat(i)]) !== null && _row_1 !== void 0 ? _row_1 : '').split(',').filter(Boolean))).sort().join(',')
+                            };
+                        })
+                    ];
+            }
+        });
+    })();
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/sqlite/fieldStats.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// One aggregate query, one result row: COUNT + GROUP_CONCAT(DISTINCT typeof()) per column, so\n// the query stays O(columns) rather than O(rows x columns). typeof()'s output already is the\n// shared vocabulary (integer/real/text -- mapValue() never binds a boolean or blob), so no\n// mapping step is needed here (contrast duckdb/fieldStats.ts). GROUP_CONCAT(DISTINCT ...)'s\n// order is not guaranteed sorted (spike-verified: 3+ distinct types come back in insertion\n// order), so the type set is sorted in JS.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, GROUP_CONCAT(DISTINCT typeof(${quoted})) FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => ({\n    field: name,\n    coverage: Number(row[`n${i}`] ?? 0),\n    type: [...new Set(((row[`t${i}`] as string) ?? '').split(',').filter(Boolean))].sort().join(','),\n  }));\n}\n"],"names":["fieldStats","conn","columns","scopeWhere","exprs","stmt","row","map","name","i","quoted","quoteIdent","prepare","join","get","field","coverage","Number","type","Set","split","filter","Boolean","sort"],"mappings":";;;;+BASsBA;;;eAAAA;;;wBATK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AASpB,SAAeA,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;;YAChFC,OAIAC,MACAC;;;;oBALAF,QAAQF,QAAQK,GAAG,CAAC,SAACC,MAAMC;wBAC/B,IAAMC,SAASC,IAAAA,oBAAU,EAACH;wBAC1B,OAAO,AAAC,SAAuBC,OAAfC,QAAO,UAA2CA,OAAnCD,GAAE,mCAA2DC,OAA1BA,QAAO,qBAA8CD,OAA3BC,QAAO,sBAAsB,OAAFD;oBACzH;oBACa;;wBAAMR,KAAKW,OAAO,CAAC,AAAC,UAA8CT,OAArCC,MAAMS,IAAI,CAAC,OAAM,sBAA+B,OAAXV;;;oBAAzEE,OAAO;oBACA;;wBAAMA,KAAKS,GAAG;;;oBAArBR,MAAO;oBACb;;wBAAOJ,QAAQK,GAAG,CAAC,SAACC,MAAMC;gCAEPH,OACGA;mCAHW;gCAC/BS,OAAOP;gCACPQ,UAAUC,QAAOX,QAAAA,GAAG,CAAC,AAAC,IAAK,OAAFG,GAAI,cAAZH,mBAAAA,QAAgB;gCACjCY,MAAM,AAAC,qBAAG,IAAIC,IAAI,EAAEb,SAAAA,GAAG,CAAC,AAAC,IAAK,OAAFG,GAAI,cAAZH,oBAAAA,SAA2B,IAAIc,KAAK,CAAC,KAAKC,MAAM,CAACC,WAAWC,IAAI,GAAGV,IAAI,CAAC;4BAC9F;;;;;IACF"}

package/dist/cjs/store/sqlite/open.js

@@ -34,6 +34,7 @@ var _indexts = require("../../config/index.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
 var _sharedts = require("../shared.js");
+var _transactionts = require("../transaction.js");
 var _connectionts = require("./connection.js");
 var _reconcilets = require("./reconcile.js");
 var _sqlfunctionsts = require("./sql-functions.js");
@@ -223,6 +224,29 @@ function storedTokenize(db) {
     var m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);
     return m ? m[1] : null;
 }
+// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
+// snippet(content, 2, ...) are documented against the first three columns and keep working
+// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
+// runs for text that needs it, and an empty string for text that does not. Shared by
+// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.
+function createContentTable(conn, tokenize) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.exec("CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '".concat(tokenize, "')"))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 // Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`
 // can't dump file text into context. Features add their own tables after the core ones.
 function ensureSchema(conn, cfg, tokenize) {
@@ -239,13 +263,9 @@ function ensureSchema(conn, cfg, tokenize) {
                     _state.sent();
                     // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL
                     // against the resolved tokenizer before this runs, so a stale table is already gone by now.
-                    // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and
-                    // snippet(content, 2, ...) are documented against the first three columns and keep working
-                    // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced
-                    // runs for text that needs it, and an empty string for text that does not.
                     return [
                         4,
-                        conn.exec("CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '".concat(tokenize, "')"))
+                        createContentTable(conn, tokenize)
                     ];
                 case 2:
                     _state.sent();
@@ -381,9 +401,9 @@ function connect(cfg) {
                 case 1:
                     _state.trys.push([
                         1,
-                        14,
+                        15,
                         ,
-                        15
+                        16
                     ]);
                     db.exec('PRAGMA journal_mode = WAL');
                     // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
@@ -411,7 +431,7 @@ function connect(cfg) {
                     tokenizeOnlyRebuild = false;
                     if (!(version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures)) return [
                         3,
-                        7
+                        8
                     ];
                     // Indexing derives from presets, so a config edit rebuilding the cache must say so and
                     // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.
@@ -428,19 +448,52 @@ function connect(cfg) {
                     changedKeys = (0, _reconcilets.changedSignatureKeys)(features !== null && features !== void 0 ? features : '', wantFeatures);
                     if (!(changedKeys.size === 1 && changedKeys.has('tokenize'))) return [
                         3,
-                        4
+                        5
                     ];
                     console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
-                    db.exec('DROP TABLE content');
+                    // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table
+                    // that was there before (old tokenizer, still queryable), never to no table at all.
+                    // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --
+                    // meta.features is only updated once rebuildContentTable (below) finishes, so a retry
+                    // re-enters this branch and repopulates from scratch either way.
+                    return [
+                        4,
+                        (0, _transactionts.withTransaction)(conn, function() {
+                            return _async_to_generator(function() {
+                                return _ts_generator(this, function(_state) {
+                                    switch(_state.label){
+                                        case 0:
+                                            return [
+                                                4,
+                                                conn.exec('DROP TABLE IF EXISTS content')
+                                            ];
+                                        case 1:
+                                            _state.sent();
+                                            return [
+                                                4,
+                                                createContentTable(conn, tokenize)
+                                            ];
+                                        case 2:
+                                            _state.sent();
+                                            return [
+                                                2
+                                            ];
+                                    }
+                                });
+                            })();
+                        })
+                    ];
+                case 4:
+                    _state.sent();
                     tokenizeOnlyRebuild = true;
                     return [
                         3,
-                        7
+                        8
                     ];
-                case 4:
+                case 5:
                     if (!(changedKeys.size === 1 && changedKeys.has('embed') && (0, _reconcilets.embedIdentityAdopted)(features !== null && features !== void 0 ? features : '', wantFeatures))) return [
                         3,
-                        6
+                        7
                     ];
                     // First sight of a resolved weight identity: the model itself hasn't changed, so
                     // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
@@ -449,13 +502,13 @@ function connect(cfg) {
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
                     ];
-                case 5:
+                case 6:
                     _state.sent();
                     return [
                         3,
-                        7
+                        8
                     ];
-                case 6:
+                case 7:
                     changed = (0, _reconcilets.signatureDiff)(features !== null && features !== void 0 ? features : '', wantFeatures);
                     console.error("sense: config change (".concat(changed, ") rebuilds the index"));
                     closed = true;
@@ -465,11 +518,11 @@ function connect(cfg) {
                         2,
                         connect(cfg)
                     ];
-                case 7:
+                case 8:
                     // Meta can lie after a crash between table creation and the signature write; the table's
                     // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild
-                    // just dropped content above, so storedTokenize sees no table and this guard is skipped
-                    // naturally rather than needing its own case.
+                    // already recreated content with this tokenize above, so stored already matches and this
+                    // guard is skipped naturally rather than needing its own case.
                     stored = storedTokenize(db);
                     if (stored !== null && stored !== tokenize) {
                         console.error('sense: cache was built with a different content tokenizer; rebuilding the index');
@@ -485,32 +538,32 @@ function connect(cfg) {
                         4,
                         ensureSchema(conn, cfg, tokenize)
                     ];
-                case 8:
+                case 9:
                     _state.sent();
                     rebuildWarnings = [];
                     if (!tokenizeOnlyRebuild) return [
                         3,
-                        11
+                        12
                     ];
                     return [
                         4,
                         (0, _reconcilets.rebuildContentTable)(conn, cfg, cfg.baseDir)
                     ];
-                case 9:
+                case 10:
                     rebuildWarnings = _state.sent();
                     return [
                         4,
                         (0, _sharedts.setMeta)(conn, 'features', wantFeatures)
                     ];
-                case 10:
-                    _state.sent();
-                    _state.label = 11;
                 case 11:
+                    _state.sent();
+                    _state.label = 12;
+                case 12:
                     return [
                         4,
                         (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
                     ];
-                case 12:
+                case 13:
                     recordedMaxMs = Number.apply(void 0, [
                         (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
                     ]);
@@ -519,7 +572,7 @@ function connect(cfg) {
                         4,
                         (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir)
                     ];
-                case 13:
+                case 14:
                     _ref1 = _state.sent(), parsed = _ref1.parsed, warnings = _ref1.warnings;
                     return [
                         2,
@@ -532,11 +585,11 @@ function connect(cfg) {
                             warnings: _to_consumable_array(rebuildWarnings).concat(_to_consumable_array(warnings))
                         }
                     ];
-                case 14:
+                case 15:
                     err = _state.sent();
                     if (!closed) db.close();
                     throw err;
-                case 15:
+                case 16:
                     return [
                         2
                     ];

package/dist/cjs/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,\n  // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.\n  // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n      // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n      // edit, an embed model change -- still takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        db.exec('DROP TABLE content');\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's\n    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n    // just dropped content above, so storedTokenize sees no table and this guard is skipped\n    // naturally rather than needing its own case.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","openSqlite","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","contentTokenize","undefined","literal","replace","exec","err","SenseError","message","storedTokenize","row","prepare","get","m","sql","match","ensureSchema","conn","tokenize","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","changedKeys","changed","stored","rebuildWarnings","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","DatabaseSync","registerFunctions","createConnection","console","error","close","changedSignatureKeys","size","has","embedIdentityAdopted","signatureDiff","rebuildContentTable","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","stmt","n","rmSync","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAepFA;eAAAA;;QAGAC;eAAAA;;QA2LGC;eAAAA;;QATMC;eAAAA;;QALAC;eAAAA;;;sBA9LY;wBACb;0BACQ;uBAEgC;wBAClC;wBACc;wBACR;4BAEA;2BACyE;8BACxE;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMJ,cAAc;AAGpB,IAAMC,iBAAiB;AAmB9B,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,IAAMI,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,IAAMC,aAAaC,IAAAA,wBAAe,EAACF;IACnC,IAAIC,eAAeE,WAAW,OAAON;IACrC,IAAMO,UAAUH,WAAWI,OAAO,CAAC,MAAM;IACzC,IAAI;QACFN,GAAGO,IAAI,CAAC;QACRP,GAAGO,IAAI,CAAC,AAAC,4EAAmF,OAARF,SAAQ;QAC5FL,GAAGO,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIC,oBAAU,CAAC,kBAAkB,AAAC,qBAA2E,OAAvDP,YAAW,8CAAmE,OAAvB,AAACM,IAAcE,OAAO,EAAC;IAC5I;IACA,OAAOL;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASM,eAAeX,EAAgB;IACtC,IAAMY,MAAMZ,GAAGa,OAAO,CAAC,wDAAwDC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,IAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeG,aAAaC,IAAgB,EAAElB,GAAW,EAAEmB,QAAgB;;YAapE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAZX;;wBAAMF,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA,wFAAwF;oBACxF,4FAA4F;oBAC5F,4FAA4F;oBAC5F,2FAA2F;oBAC3F,2FAA2F;oBAC3F,2EAA2E;oBAC3E;;wBAAMY,KAAKZ,IAAI,CAAC,AAAC,6IAAqJ,OAATa,UAAS;;;oBAAtK;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMY,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBe,IAAAA,wBAAc,EAACrB;;;2BAAhC,6BAAA,QAAA;;;;oBAAMoB,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACJ;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMK,IAAAA,iBAAO,EAACL,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM,kBAAkBzB;;;oBAAtC;;;oBACjD;;wBAAM8B,IAAAA,iBAAO,EAACL,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM,YAAYO,IAAAA,yBAAgB,EAACzB,KAAK0B,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQ3B,GAAmB;;YAClC4B,UAEAC,QAEA9B,IAIF+B,QA6E4B,MAtExBZ,MAMAC,UAIAY,SACAC,UACAC,cACFC,qBAWIC,aAcEC,SAaJC,QAWFC,iBAQEC,eAGuB,OAArBC,QAAQC,UAGTlC;;;;oBA3FHqB,WAAWc,IAAAA,cAAI,EAAC1C,IAAI2C,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACjB,UAAU;wBAAEkB,WAAW;oBAAK;oBAChCjB,SAASa,IAAAA,cAAI,EAACd,UAAUpC;oBAExBO,KAAK,IAAIgD,wBAAY,CAAClB;oBAC5B,8FAA8F;oBAC9F,6FAA6F;oBAC7F,8FAA8F;oBAC1FC,SAAS;;;;;;;;;oBAEX/B,GAAGO,IAAI,CAAC;oBACR,yFAAyF;oBACzF,uCAAuC;oBACvCP,GAAGO,IAAI,CAAC;oBACR0C,IAAAA,iCAAiB,EAACjD,IAAIG,IAAAA,wBAAe,EAACF,SAASG;oBACzCe,OAAO+B,IAAAA,8BAAgB,EAAClD;oBAE9BA,GAAGO,IAAI,CAAC;oBAER,2FAA2F;oBAC3F,8FAA8F;oBACxFa,WAAWrB,gBAAgBC,IAAIC;oBAIrB;;wBAAMuB,IAAAA,iBAAO,EAACL,MAAM;;;oBAA9Ba,UAAU;oBACC;;wBAAMR,IAAAA,iBAAO,EAACL,MAAM;;;oBAA/Bc,WAAW;oBACXC,eAAeR,IAAAA,yBAAgB,EAACzB,KAAK0B,kBAAQ;oBAC/CQ,sBAAsB;yBACtB,CAAA,AAACH,YAAY,QAAQA,YAAYtC,kBAAoBuC,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,uFAAuF;oBACvF,wFAAwF;oBACxF,IAAIF,YAAY,QAAQA,YAAYtC,gBAAgB;wBAClDyD,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACT/B,GAAGqD,KAAK;wBACR1D,WAAWM;wBACX;;4BAAO2B,QAAQ3B;;oBACjB;oBACMmC,cAAckB,IAAAA,iCAAoB,EAACrB,qBAAAA,sBAAAA,WAAY,IAAIC;yBAIrDE,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,WAAU,GAApDpB;;;;oBACFe,QAAQC,KAAK,CAAC;oBACdpD,GAAGO,IAAI,CAAC;oBACR4B,sBAAsB;;;;;;yBACbC,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,YAAYC,IAAAA,iCAAoB,EAACxB,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGE;;;;oBACT,iFAAiF;oBACjF,wFAAwF;oBACxFe,QAAQC,KAAK,CAAC;oBACd;;wBAAM3B,IAAAA,iBAAO,EAACN,MAAM,YAAYe;;;oBAAhC;;;;;;oBAEMG,UAAUqB,IAAAA,0BAAa,EAACzB,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9CiB,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARf,SAAQ;oBAC/CN,SAAS;oBACT/B,GAAGqD,KAAK;oBACR1D,WAAWM;oBACX;;wBAAO2B,QAAQ3B;;;oBAInB,yFAAyF;oBACzF,6FAA6F;oBAC7F,wFAAwF;oBACxF,8CAA8C;oBACxCqC,SAAS3B,eAAeX;oBAC9B,IAAIsC,WAAW,QAAQA,WAAWlB,UAAU;wBAC1C+B,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACT/B,GAAGqD,KAAK;wBACR1D,WAAWM;wBACX;;4BAAO2B,QAAQ3B;;oBACjB;oBAEA;;wBAAMiB,aAAaC,MAAMlB,KAAKmB;;;oBAA9B;oBAEImB;yBACAJ,qBAAAA;;;;oBACgB;;wBAAMwB,IAAAA,gCAAmB,EAACxC,MAAMlB,KAAKA,IAAI2C,OAAO;;;oBAAlEL,kBAAkB;oBAClB;;wBAAMd,IAAAA,iBAAO,EAACN,MAAM,YAAYe;;;oBAAhC;;;oBAK4B;;wBAAMV,IAAAA,iBAAO,EAACL,MAAM;;;oBAA5CqB,gBAAgBoB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E5D,GAAGO,IAAI,CAAC,AAAC,yBAA8E,OAAtDsD,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIvB,gBAAgB;oBAEjD;;wBAAMwB,IAAAA,sBAAS,EAAC7C,MAAMlB,KAAKA,IAAI2C,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAE1C,IAAAA;4BAAImB,MAAAA;4BAAMlB,KAAAA;4BAAK6B,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAU,AAAC,qBAAGH,wBAAiB,qBAAGG;wBAAU;;;oBAC7ElC;oBACP,IAAI,CAACuB,QAAQ/B,GAAGqD,KAAK;oBACrB,MAAM7C;;;;;;;IAEV;;AAIO,SAAeX,WAAWI,GAAmB;;YACe,MAAzDD,IAAImB,MAAW8C,aAAanC,QAAQW,QAAQC;;;;oBAAa;;wBAAMd,QAAQ3B;;;oBAAd,OAAA,eAAzDD,KAAyD,KAAzDA,IAAImB,OAAqD,KAArDA,MAAW8C,cAA0C,KAA/ChE,KAAkB6B,SAA6B,KAA7BA,QAAQW,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBACpD;;wBAAO;4BAAEwB,OAAOC,IAAAA,oBAAW,EAACnE,IAAImB,MAAM8C,aAAaA,YAAYrB,OAAO;4BAAG3C,KAAKgE;4BAAanC,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACtH;;AAEO,SAAe9C,SAASsE,KAAY;;YACnCE;;;;oBAAO;;wBAAMF,MAAMrD,OAAO,CAAC;;;oBAA3BuD,OAAO;oBACJ;;wBAAMA,KAAKtD,GAAG;;;oBAAvB;;wBAAS,cAAoCuD,CAAC;;;;IAChD;;AAMO,SAAS1E,WAAWM,GAAmB;IAC5CqE,IAAAA,cAAM,EAAC3B,IAAAA,cAAI,EAAC1C,IAAI2C,OAAO,EAAEC,kBAAS,GAAG;QAAEE,WAAW;QAAMwB,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n// snippet(content, 2, ...) are documented against the first three columns and keep working\n// (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n// runs for text that needs it, and an empty string for text that does not. Shared by\n// ensureSchema and the tokenize-only rebuild in connect(), so both create identical DDL.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below (tokenizer probe, schema, reconcile's COLUMN_LIMIT) must release this handle,\n  // or on Windows the leaked WAL db is undeletable and scratch cleanup fails with EPERM/EBUSY.\n  // closed marks the rebuild branches that close-and-recurse, so the catch never double-closes.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n      // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n      // edit, an embed model change -- still takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // Drop and recreate as one transaction: a crash before COMMIT rolls back to the table\n        // that was there before (old tokenizer, still queryable), never to no table at all.\n        // IF EXISTS also makes a retry after such a crash a no-op instead of a raw SQLite error --\n        // meta.features is only updated once rebuildContentTable (below) finishes, so a retry\n        // re-enters this branch and repopulates from scratch either way.\n        await withTransaction(conn, async () => {\n          await conn.exec('DROP TABLE IF EXISTS content');\n          await createContentTable(conn, tokenize);\n        });\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's\n    // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n    // already recreated content with this tokenize above, so stored already matches and this\n    // guard is skipped naturally rather than needing its own case.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  return { store: createStore(db, conn, resolvedCfg, resolvedCfg.baseDir), cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","openSqlite","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","contentTokenize","undefined","literal","replace","exec","err","SenseError","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","activeFeatures","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","changedKeys","changed","stored","rebuildWarnings","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","DatabaseSync","registerFunctions","createConnection","console","error","close","changedSignatureKeys","size","has","withTransaction","embedIdentityAdopted","signatureDiff","rebuildContentTable","Number","Math","min","max","reconcile","resolvedCfg","store","createStore","stmt","n","rmSync","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;;;;;;;;;;;;QAgBpFA;eAAAA;;QAGAC;eAAAA;;QAwMGC;eAAAA;;QATMC;eAAAA;;QALAC;eAAAA;;;sBA5MY;wBACb;0BACQ;uBAEgC;wBAClC;wBACc;wBACR;6BACD;4BAEC;2BACyE;8BACxE;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMJ,cAAc;AAGpB,IAAMC,iBAAiB;AAmB9B,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,IAAMI,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,IAAMC,aAAaC,IAAAA,wBAAe,EAACF;IACnC,IAAIC,eAAeE,WAAW,OAAON;IACrC,IAAMO,UAAUH,WAAWI,OAAO,CAAC,MAAM;IACzC,IAAI;QACFN,GAAGO,IAAI,CAAC;QACRP,GAAGO,IAAI,CAAC,AAAC,4EAAmF,OAARF,SAAQ;QAC5FL,GAAGO,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIC,oBAAU,CAAC,kBAAkB,AAAC,qBAA2E,OAAvDP,YAAW,8CAAmE,OAAvB,AAACM,IAAcE,OAAO,EAAC;IAC5I;IACA,OAAOL;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASM,eAAeX,EAAgB;IACtC,IAAMY,MAAMZ,GAAGa,OAAO,CAAC,wDAAwDC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,IAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,4FAA4F;AAC5F,2FAA2F;AAC3F,2FAA2F;AAC3F,qFAAqF;AACrF,yFAAyF;AACzF,SAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;;;;;oBAClE;;wBAAMD,KAAKZ,IAAI,CAAC,AAAC,6IAAqJ,OAATa,UAAS;;;oBAAtK;;;;;;IACF;;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAAeC,aAAaF,IAAgB,EAAElB,GAAW,EAAEmB,QAAgB;;YASpE,2BAAA,mBAAA,gBAAA,WAAA,OAAME;;;;oBARX;;wBAAMH,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA,wFAAwF;oBACxF,4FAA4F;oBAC5F;;wBAAMW,mBAAmBC,MAAMC;;;oBAA/B;oBACA,6FAA6F;oBAC7F,qFAAqF;oBACrF;;wBAAMD,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMY,KAAKZ,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBgB,IAAAA,wBAAc,EAACtB;;;2BAAhC,6BAAA,QAAA;;;;oBAAMqB,UAAN;oBAAsC;;wBAAMA,QAAQE,MAAM,CAACL;;;oBAArB;;;oBAAtC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACA;;wBAAMM,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,kBAAkBzB;;;oBAAtC;;;oBACjD;;wBAAM+B,IAAAA,iBAAO,EAACN,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMO,IAAAA,iBAAO,EAACP,MAAM,YAAYQ,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,QAAQ5B,GAAmB;;YAClC6B,UAEAC,QAEA/B,IAIFgC,QAqF4B,MA9ExBb,MAMAC,UAIAa,SACAC,UACAC,cACFC,qBAWIC,aAsBEC,SAaJC,QAWFC,iBAQEC,eAGuB,OAArBC,QAAQC,UAGTnC;;;;oBAnGHsB,WAAWc,IAAAA,cAAI,EAAC3C,IAAI4C,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAACjB,UAAU;wBAAEkB,WAAW;oBAAK;oBAChCjB,SAASa,IAAAA,cAAI,EAACd,UAAUrC;oBAExBO,KAAK,IAAIiD,wBAAY,CAAClB;oBAC5B,8FAA8F;oBAC9F,6FAA6F;oBAC7F,8FAA8F;oBAC1FC,SAAS;;;;;;;;;oBAEXhC,GAAGO,IAAI,CAAC;oBACR,yFAAyF;oBACzF,uCAAuC;oBACvCP,GAAGO,IAAI,CAAC;oBACR2C,IAAAA,iCAAiB,EAAClD,IAAIG,IAAAA,wBAAe,EAACF,SAASG;oBACzCe,OAAOgC,IAAAA,8BAAgB,EAACnD;oBAE9BA,GAAGO,IAAI,CAAC;oBAER,2FAA2F;oBAC3F,8FAA8F;oBACxFa,WAAWrB,gBAAgBC,IAAIC;oBAIrB;;wBAAMwB,IAAAA,iBAAO,EAACN,MAAM;;;oBAA9Bc,UAAU;oBACC;;wBAAMR,IAAAA,iBAAO,EAACN,MAAM;;;oBAA/Be,WAAW;oBACXC,eAAeR,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;oBAC/CQ,sBAAsB;yBACtB,CAAA,AAACH,YAAY,QAAQA,YAAYvC,kBAAoBwC,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,uFAAuF;oBACvF,wFAAwF;oBACxF,IAAIF,YAAY,QAAQA,YAAYvC,gBAAgB;wBAClD0D,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACThC,GAAGsD,KAAK;wBACR3D,WAAWM;wBACX;;4BAAO4B,QAAQ5B;;oBACjB;oBACMoC,cAAckB,IAAAA,iCAAoB,EAACrB,qBAAAA,sBAAAA,WAAY,IAAIC;yBAIrDE,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,WAAU,GAApDpB;;;;oBACFe,QAAQC,KAAK,CAAC;oBACd,sFAAsF;oBACtF,oFAAoF;oBACpF,2FAA2F;oBAC3F,sFAAsF;oBACtF,iEAAiE;oBACjE;;wBAAMK,IAAAA,8BAAe,EAACvC,MAAM;;;;;4CAC1B;;gDAAMA,KAAKZ,IAAI,CAAC;;;4CAAhB;4CACA;;gDAAMW,mBAAmBC,MAAMC;;;4CAA/B;;;;;;4BACF;;;;oBAHA;oBAIAgB,sBAAsB;;;;;;yBACbC,CAAAA,YAAYmB,IAAI,KAAK,KAAKnB,YAAYoB,GAAG,CAAC,YAAYE,IAAAA,iCAAoB,EAACzB,qBAAAA,sBAAAA,WAAY,IAAIC,aAAY,GAAvGE;;;;oBACT,iFAAiF;oBACjF,wFAAwF;oBACxFe,QAAQC,KAAK,CAAC;oBACd;;wBAAM3B,IAAAA,iBAAO,EAACP,MAAM,YAAYgB;;;oBAAhC;;;;;;oBAEMG,UAAUsB,IAAAA,0BAAa,EAAC1B,qBAAAA,sBAAAA,WAAY,IAAIC;oBAC9CiB,QAAQC,KAAK,CAAC,AAAC,yBAAgC,OAARf,SAAQ;oBAC/CN,SAAS;oBACThC,GAAGsD,KAAK;oBACR3D,WAAWM;oBACX;;wBAAO4B,QAAQ5B;;;oBAInB,yFAAyF;oBACzF,6FAA6F;oBAC7F,yFAAyF;oBACzF,+DAA+D;oBACzDsC,SAAS5B,eAAeX;oBAC9B,IAAIuC,WAAW,QAAQA,WAAWnB,UAAU;wBAC1CgC,QAAQC,KAAK,CAAC;wBACdrB,SAAS;wBACThC,GAAGsD,KAAK;wBACR3D,WAAWM;wBACX;;4BAAO4B,QAAQ5B;;oBACjB;oBAEA;;wBAAMoB,aAAaF,MAAMlB,KAAKmB;;;oBAA9B;oBAEIoB;yBACAJ,qBAAAA;;;;oBACgB;;wBAAMyB,IAAAA,gCAAmB,EAAC1C,MAAMlB,KAAKA,IAAI4C,OAAO;;;oBAAlEL,kBAAkB;oBAClB;;wBAAMd,IAAAA,iBAAO,EAACP,MAAM,YAAYgB;;;oBAAhC;;;oBAK4B;;wBAAMV,IAAAA,iBAAO,EAACN,MAAM;;;oBAA5CsB,gBAAgBqB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E9D,GAAGO,IAAI,CAAC,AAAC,yBAA8E,OAAtDwD,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIxB,gBAAgB;oBAEjD;;wBAAMyB,IAAAA,sBAAS,EAAC/C,MAAMlB,KAAKA,IAAI4C,OAAO;;;oBAAtC,QAAA,eAArBH,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAE3C,IAAAA;4BAAImB,MAAAA;4BAAMlB,KAAAA;4BAAK8B,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAU,AAAC,qBAAGH,wBAAiB,qBAAGG;wBAAU;;;oBAC7EnC;oBACP,IAAI,CAACwB,QAAQhC,GAAGsD,KAAK;oBACrB,MAAM9C;;;;;;;IAEV;;AAIO,SAAeX,WAAWI,GAAmB;;YACe,MAAzDD,IAAImB,MAAWgD,aAAapC,QAAQW,QAAQC;;;;oBAAa;;wBAAMd,QAAQ5B;;;oBAAd,OAAA,eAAzDD,KAAyD,KAAzDA,IAAImB,OAAqD,KAArDA,MAAWgD,cAA0C,KAA/ClE,KAAkB8B,SAA6B,KAA7BA,QAAQW,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBACpD;;wBAAO;4BAAEyB,OAAOC,IAAAA,oBAAW,EAACrE,IAAImB,MAAMgD,aAAaA,YAAYtB,OAAO;4BAAG5C,KAAKkE;4BAAapC,QAAAA;4BAAQW,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACtH;;AAEO,SAAe/C,SAASwE,KAAY;;YACnCE;;;;oBAAO;;wBAAMF,MAAMvD,OAAO,CAAC;;;oBAA3ByD,OAAO;oBACJ;;wBAAMA,KAAKxD,GAAG;;;oBAAvB;;wBAAS,cAAoCyD,CAAC;;;;IAChD;;AAMO,SAAS5E,WAAWM,GAAmB;IAC5CuE,IAAAA,cAAM,EAAC5B,IAAAA,cAAI,EAAC3C,IAAI4C,OAAO,EAAEC,kBAAS,GAAG;QAAEE,WAAW;QAAMyB,OAAO;IAAK;AACtE"}

package/dist/cjs/store/sqlite/store.js

@@ -19,9 +19,11 @@ _export(exports, {
 var _indexts = require("../../config/index.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
+var _vectorsts = require("../vectors.js");
+var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
 var _reconcilets = require("./reconcile.js");
-var _vectorsts = require("./vectors.js");
+var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
     for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
@@ -457,6 +459,16 @@ function createStore(db, conn, cfg, baseDir) {
                         }
                     });
                 })();
+            },
+            fieldStats: function fieldStats(columns, scopeWhere) {
+                return _async_to_generator(function() {
+                    return _ts_generator(this, function(_state) {
+                        return [
+                            2,
+                            (0, _fieldStatsts.fieldStats)(conn, columns, scopeWhere)
+                        ];
+                    });
+                })();
             }
         },
         lexical: {
@@ -489,7 +501,7 @@ function createStore(db, conn, cfg, baseDir) {
                             case 0:
                                 return [
                                     4,
-                                    (0, _vectorsts.writeVectorBatch)(conn, rows)
+                                    (0, _vectorsts1.writeVectorBatch)(conn, rows)
                                 ];
                             case 1:
                                 _state.sent();
@@ -505,7 +517,7 @@ function createStore(db, conn, cfg, baseDir) {
                     return _ts_generator(this, function(_state) {
                         return [
                             2,
-                            (0, _vectorsts.scanCandidates)(conn, qv, storeDims, fetch, allowed)
+                            (0, _vectorsts1.scanCandidates)(conn, qv, storeDims, fetch, allowed)
                         ];
                     });
                 })();
@@ -515,7 +527,7 @@ function createStore(db, conn, cfg, baseDir) {
                     return _ts_generator(this, function(_state) {
                         return [
                             2,
-                            (0, _vectorsts.scanSimilar)(conn, path, opts)
+                            (0, _vectorsts1.scanSimilar)(conn, path, opts)
                         ];
                     });
                 })();
@@ -531,6 +543,22 @@ function createStore(db, conn, cfg, baseDir) {
                 })();
             }
         },
+        engineStatus: function engineStatus() {
+            return _async_to_generator(function() {
+                var row;
+                return _ts_generator(this, function(_state) {
+                    // Read back rather than recomputed: this is what open() actually set (3x the largest
+                    // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.
+                    row = db.prepare('PRAGMA busy_timeout').get();
+                    return [
+                        2,
+                        {
+                            busy_timeout: "".concat(row.timeout, "ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)")
+                        }
+                    ];
+                });
+            })();
+        },
         raw: {
             prepare: function prepare(sql) {
                 return _async_to_generator(function() {

package/dist/cjs/store/sqlite/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { hasVectorRow, pendingRows, scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth (see\n// transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","docs","columns","getColumns","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAUaA;eAAAA;;QAKGC;eAAAA;;;uBAbgB;wBACL;6BACK;yBAEH;2BACH;yBAC+D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAElF,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU;AAKxH,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC;;;YAC9B;;QACAY,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAAChB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;QACF;QACAiB,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACrB,MAAMmB,OAAOC,MAAME,IAAAA,wBAAe,EAACrB,SAASsB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC1B;;;gBACrB;;YACM2B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,2BAAgB,EAAC7B,MAAM4B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,yBAAc,EAACnC,MAAM+B,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,sBAAW,EAACtC,MAAMqC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAACxC,MAAMqC;;;gBAC5B;;QACF;QACAI,KAAK;YACGlC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBoC;;wBAAAA,OAAO3C,GAAGQ,OAAO,CAACD;wBACxBoC,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACL5B,SAAS,SAATA;2CAAe2B,KAAK3B,OAAO;;gCAC3B,2EAA2E;gCAC3E6B,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ/C,GAAG+C,KAAK;;;;;YACV;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/store.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store, VectorWriteRow } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors']);\n\n// Wraps the synchronous DatabaseSync connection in the async Store interface, sharing one\n// Connection instance (conn) with open()'s own reconcile call so transaction depth (see\n// transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: DatabaseSync, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'sqlite',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      async fieldStats(columns, scopeWhere) {\n        return fieldStats(conn, columns, scopeWhere);\n      },\n    },\n    lexical: {\n      async query(terms, opts) {\n        return queryLexical(conn, terms, opts, contentTokenize(cfg) === undefined);\n      },\n    },\n    vectors: {\n      async pending() {\n        return pendingRows(conn);\n      },\n      async writeVectors(rows: VectorWriteRow[]) {\n        await writeVectorBatch(conn, rows);\n      },\n      async candidates(qv, storeDims, fetch, allowed) {\n        return scanCandidates(conn, qv, storeDims, fetch, allowed);\n      },\n      async similar(path, opts) {\n        return scanSimilar(conn, path, opts);\n      },\n      async hasVector(path) {\n        return hasVectorRow(conn, path);\n      },\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min), not a value re-derived here.\n      const row = db.prepare('PRAGMA busy_timeout').get() as { timeout: number };\n      return { busy_timeout: `${row.timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = db.prepare(sql);\n        stmt.setReadBigInts(true); // int64 past 2^53 arrives as BigInt instead of throwing at step time\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through an async iterator wrapping the sync iterate().\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...(params as Parameters<typeof stmt.iterate>));\n          },\n        };\n      },\n    },\n    async close() {\n      db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","contentTokenize","undefined","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","candidates","qv","storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","timeout","raw","stmt","setReadBigInts","iterate","params","close"],"mappings":";;;;;;;;;;;QAYaA;eAAAA;;QAKGC;eAAAA;;;uBAfgB;wBACL;6BACK;yBAEU;4BACf;yBACE;2BACH;0BACoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAY;IAAqB;IAAW;CAAU;AAKxH,SAASD,YAAYE,EAAgB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC1F,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC;;;YAC9B;;QACAY,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAAChB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACMiB,YAAN,SAAMA,WAAWF,OAAO,EAAEG,UAAU;;;wBAClC;;4BAAOD,IAAAA,wBAAU,EAACjB,MAAMe,SAASG;;;gBACnC;;QACF;QACAC,SAAS;YACDC,OAAN,SAAMA,MAAMC,KAAK,EAAEC,IAAI;;;wBACrB;;4BAAOC,IAAAA,uBAAY,EAACvB,MAAMqB,OAAOC,MAAME,IAAAA,wBAAe,EAACvB,SAASwB;;;gBAClE;;QACF;QACAC,SAAS;YACDC,SAAN,SAAMA;;;wBACJ;;4BAAOC,IAAAA,sBAAW,EAAC5B;;;gBACrB;;YACM6B,cAAN,SAAMA,aAAaC,IAAsB;;;;;gCACvC;;oCAAMC,IAAAA,4BAAgB,EAAC/B,MAAM8B;;;gCAA7B;;;;;;gBACF;;YACME,YAAN,SAAMA,WAAWC,EAAE,EAAEC,SAAS,EAAEC,KAAK,EAAEC,OAAO;;;wBAC5C;;4BAAOC,IAAAA,0BAAc,EAACrC,MAAMiC,IAAIC,WAAWC,OAAOC;;;gBACpD;;YACME,SAAN,SAAMA,QAAQC,IAAI,EAAEjB,IAAI;;;wBACtB;;4BAAOkB,IAAAA,uBAAW,EAACxC,MAAMuC,MAAMjB;;;gBACjC;;YACMmB,WAAN,SAAMA,UAAUF,IAAI;;;wBAClB;;4BAAOG,IAAAA,uBAAY,EAAC1C,MAAMuC;;;gBAC5B;;QACF;QACMI,cAAN,SAAMA;;oBAGEC;;oBAFN,qFAAqF;oBACrF,qFAAqF;oBAC/EA,MAAM7C,GAAGQ,OAAO,CAAC,uBAAuBsC,GAAG;oBACjD;;wBAAO;4BAAEC,cAAc,AAAC,GAAc,OAAZF,IAAIG,OAAO,EAAC;wBAAoF;;;YAC5H;;QACAC,KAAK;YACGzC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB2C;;wBAAAA,OAAOlD,GAAGQ,OAAO,CAACD;wBACxB2C,KAAKC,cAAc,CAAC,OAAO,qEAAqE;wBAChG;;4BAAO;gCACLnC,SAAS,SAATA;2CAAekC,KAAKlC,OAAO;;gCAC3B,2EAA2E;gCAC3EoC,SAAS,SAATA;oCAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;wCAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;4CACrCH;;;;oDAAP;;mEAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAIG;;;oDAAxB;;;;;;oCACF;;4BACF;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJtD,GAAGsD,KAAK;;;;;YACV;;IACF;AACF"}