sensemaking 0.19.2 → 0.21.0

package/dist/cjs/store/turso/open.js

@@ -9,12 +9,6 @@ function _export(target, all) {
     });
 }
 _export(exports, {
-    get CONTENT_FTS_DDL () {
-        return CONTENT_FTS_DDL;
-    },
-    get CONTENT_FTS_NAMES () {
-        return CONTENT_FTS_NAMES;
-    },
     get DB_FILENAME () {
         return DB_FILENAME;
     },
@@ -23,21 +17,20 @@ _export(exports, {
     },
     get openTurso () {
         return openTurso;
+    },
+    get tursoOpenDialect () {
+        return tursoOpenDialect;
     }
 });
-var _nodefs = require("node:fs");
-var _nodepath = require("node:path");
 var _indexts = require("../../config/index.js");
-var _handoffts = require("../../embed/handoff.js");
 var _typests = require("../../embed/types.js");
 var _errorsts = require("../../errors.js");
 var _indexts1 = require("../../features/index.js");
-var _cachets = require("../cache.js");
-var _reconcilets = require("../reconcile.js");
+var _opents = require("../open.js");
 var _sharedts = require("../shared.js");
 var _connectionts = require("./connection.js");
 var _nativets = require("./native.js");
-var _reconcilets1 = require("./reconcile.js");
+var _reconcilets = require("./reconcile.js");
 var _storets = require("./store.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
@@ -171,16 +164,10 @@ function _ts_generator(thisArg, body) {
     }
 }
 var DB_FILENAME = 'cache.turso.db';
-var SCHEMA_VERSION = '3';
-var CONTENT_FTS_DDL = [
-    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
-    "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
-];
-var CONTENT_FTS_NAMES = [
-    'content_fts',
-    'content_fts_ngram'
-];
-function ensureSchema(conn, cfg) {
+var SCHEMA_VERSION = '4';
+// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
+// extra native state to thread, unlike duckdb's separate instance/connection pair.
+function ensureSchema(_handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, ddl, err, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, feature, err;
         return _ts_generator(this, function(_state) {
@@ -207,7 +194,7 @@ function ensureSchema(conn, cfg) {
                         9,
                         10
                     ]);
-                    _iterator = CONTENT_FTS_DDL[Symbol.iterator]();
+                    _iterator = _reconcilets.CONTENT_FTS_DDL[Symbol.iterator]();
                     _state.label = 4;
                 case 4:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
@@ -380,45 +367,74 @@ function ensureSchema(conn, cfg) {
         });
     })();
 }
-function connect(cfg) {
+function close(handle) {
     return _async_to_generator(function() {
-        var stateDir, dbPath, turso, err, db, err1, closed, _ref, conn, version, features, wantFeatures, reason, recordedMaxMs, _ref1, parsed, warnings, err2;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    stateDir = (0, _nodepath.join)(cfg.baseDir, _indexts.STATE_DIR);
-                    (0, _nodefs.mkdirSync)(stateDir, {
-                        recursive: true
-                    });
-                    dbPath = (0, _nodepath.join)(stateDir, DB_FILENAME);
-                    _state.label = 1;
+                    return [
+                        4,
+                        handle.close()
+                    ];
                 case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function setDerivedBusyTimeout(_handle, conn, ms) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    return [
+                        4,
+                        conn.exec("PRAGMA busy_timeout = ".concat(ms))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+function connect(dbPath, _cfg) {
+    return _async_to_generator(function() {
+        var turso, err, db, err1;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
                     _state.trys.push([
-                        1,
-                        3,
+                        0,
+                        2,
                         ,
-                        4
+                        3
                     ]);
                     return [
                         4,
                         (0, _nativets.tursoApi)()
                     ];
-                case 2:
+                case 1:
                     turso = _state.sent();
                     return [
                         3,
-                        4
+                        3
                     ];
-                case 3:
+                case 2:
                     err = _state.sent();
                     if (_instanceof(err, _errorsts.SenseError)) throw err;
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" needs the '.concat(_nativets.TURSO_PACKAGE, " package (").concat(err.message, ")"));
-                case 4:
+                case 3:
                     _state.trys.push([
-                        4,
-                        6,
+                        3,
+                        5,
                         ,
-                        7
+                        6
                     ]);
                     return [
                         4,
@@ -429,156 +445,54 @@ function connect(cfg) {
                             ]
                         })
                     ];
-                case 5:
+                case 4:
                     // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
                     // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
                     db = _state.sent();
                     return [
                         3,
-                        7
+                        6
                     ];
-                case 6:
+                case 5:
                     err1 = _state.sent();
                     throw new _errorsts.SenseError('STORE_DEPENDENCY_MISSING', 'store "turso" failed to open '.concat(dbPath, ": ").concat(err1.message));
-                case 7:
-                    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
-                    // Windows. closed guards the catch against double-closing.
-                    closed = false;
-                    _state.label = 8;
-                case 8:
-                    _state.trys.push([
-                        8,
-                        18,
-                        ,
-                        21
-                    ]);
-                    conn = (0, _connectionts.createConnection)(db);
-                    return [
-                        4,
-                        conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)')
-                    ];
-                case 9:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'schema_version')
-                    ];
-                case 10:
-                    version = _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'features')
-                    ];
-                case 11:
-                    features = _state.sent();
-                    wantFeatures = (0, _indexts.featureSignature)(cfg, _indexts1.FEATURES);
-                    if (!(version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures)) return [
-                        3,
-                        13
-                    ];
-                    // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
-                    // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
-                    reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-                    console.error("sense: ".concat(reason, "; rebuilding the index"));
-                    closed = true;
-                    return [
-                        4,
-                        db.close()
-                    ];
-                case 12:
-                    _state.sent();
-                    (0, _cachets.clearCache)(cfg);
-                    return [
-                        2,
-                        connect(cfg)
-                    ];
-                case 13:
-                    return [
-                        4,
-                        ensureSchema(conn, cfg)
-                    ];
-                case 14:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _sharedts.getMeta)(conn, 'reconcile_max_ms')
-                    ];
-                case 15:
-                    recordedMaxMs = Number.apply(void 0, [
-                        (_ref = _state.sent()) !== null && _ref !== void 0 ? _ref : '0'
-                    ]);
-                    return [
-                        4,
-                        conn.exec("PRAGMA busy_timeout = ".concat(Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)))
-                    ];
-                case 16:
-                    _state.sent();
-                    return [
-                        4,
-                        (0, _reconcilets.reconcile)(conn, cfg, cfg.baseDir, _reconcilets1.tursoDialect)
-                    ];
-                case 17:
-                    _ref1 = _state.sent(), parsed = _ref1.parsed, warnings = _ref1.warnings;
+                case 6:
                     return [
                         2,
                         {
-                            db: db,
-                            conn: conn,
-                            cfg: cfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
+                            handle: db,
+                            conn: (0, _connectionts.createConnection)(db)
                         }
                     ];
-                case 18:
-                    err2 = _state.sent();
-                    if (!!closed) return [
-                        3,
-                        20
-                    ];
-                    return [
-                        4,
-                        db.close()
-                    ];
-                case 19:
-                    _state.sent();
-                    _state.label = 20;
-                case 20:
-                    throw err2;
-                case 21:
-                    return [
-                        2
-                    ];
             }
         });
     })();
 }
+var tursoOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: _reconcilets.tursoDialect,
+    connect: connect,
+    close: close,
+    // "Locking error: Failed locking file ...", worded per platform: posix "File is locked by another
+    // process", Windows "another process has locked a portion of the file (os error 33)". Distinct from
+    // the write-time "database is locked" its connect-time `timeout` covers; that one never reaches here.
+    isLocked: function isLocked(err) {
+        return /File is locked by another process|locked a portion of the file/.test(err.message);
+    },
+    ensureSchema: ensureSchema,
+    setDerivedBusyTimeout: setDerivedBusyTimeout,
+    createStore: function createStore(handle, conn) {
+        return (0, _storets.createStore)(handle, conn);
+    }
+};
 function openTurso(cfg) {
     return _async_to_generator(function() {
-        var _ref, db, conn, resolvedCfg, dbPath, parsed, warnings, store;
         return _ts_generator(this, function(_state) {
-            switch(_state.label){
-                case 0:
-                    return [
-                        4,
-                        connect(cfg)
-                    ];
-                case 1:
-                    _ref = _state.sent(), db = _ref.db, conn = _ref.conn, resolvedCfg = _ref.cfg, dbPath = _ref.dbPath, parsed = _ref.parsed, warnings = _ref.warnings;
-                    store = (0, _storets.createStore)(db, conn, resolvedCfg, resolvedCfg.baseDir);
-                    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-                    (0, _handoffts.rekeyChunkText)(conn, store);
-                    return [
-                        2,
-                        {
-                            store: store,
-                            cfg: resolvedCfg,
-                            dbPath: dbPath,
-                            parsed: parsed,
-                            warnings: warnings
-                        }
-                    ];
-            }
+            return [
+                2,
+                (0, _opents.openWithDialect)(cfg, tursoOpenDialect)
+            ];
         });
     })();
 }

package/dist/cjs/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","DB_FILENAME","SCHEMA_VERSION","openTurso","ensureSchema","conn","cfg","ddl","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","connect","stateDir","dbPath","turso","err","db","closed","version","features","wantFeatures","reason","recordedMaxMs","parsed","warnings","join","baseDir","STATE_DIR","mkdirSync","recursive","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","console","error","close","clearCache","Number","Math","min","max","reconcile","tursoDialect","resolvedCfg","store","createStore","rekeyChunkText"],"mappings":";;;;;;;;;;;QAmCaA;eAAAA;;QAIAC;eAAAA;;QArBAC;eAAAA;;QAGAC;eAAAA;;QA+GSC;eAAAA;;;sBApII;wBACL;uBAGuB;yBACb;uBACJ;wBACA;wBACc;uBACd;2BACD;wBACO;4BAEA;wBACO;4BACX;uBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMF,cAAc;AAGpB,IAAMC,iBAAiB;AAcvB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,SAAeI,aAAaC,IAAgB,EAAEC,GAAW;;YAGlD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaV;;;2BAAb,6BAAA,QAAA;;;;oBAAMQ,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBC,IAAAA,wBAAc,EAACJ;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXG,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACR;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,kBAAkBH;;;oBAAtC;;;oBACjD;;wBAAMY,IAAAA,iBAAO,EAACT,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM,YAAYW,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAWA,SAAeC,QAAQZ,GAAmB;;YAClCa,UAEAC,QAIFC,OAGKC,KAKLC,IAKKD,MAKLE,QAwB4B,MAtBxBnB,MAIAoB,SACAC,UACAC,cAIEC,QAYFC,eAGuB,OAArBC,QAAQC,UAGTT;;;;oBAtDHH,WAAWa,IAAAA,cAAI,EAAC1B,IAAI2B,OAAO,EAAEC,kBAAS;oBAC5CC,IAAAA,iBAAS,EAAChB,UAAU;wBAAEiB,WAAW;oBAAK;oBAChChB,SAASY,IAAAA,cAAI,EAACb,UAAUlB;;;;;;;;;oBAMpB;;wBAAMoC,IAAAA,kBAAQ;;;oBAAtBhB,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAegB,oBAAU,GAAE,MAAMhB;oBACrC,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACjB,IAAckB,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMnB,MAAMH,OAAO,CAACE,QAAQ;4BAAEqB,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHnB,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIgB,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXlB,QAAO,MAA2B,OAAvB,AAACE,KAAckB,OAAO;;oBAEpH,6FAA6F;oBAC7F,2DAA2D;oBACvDhB,SAAS;;;;;;;;;oBAELnB,OAAOsC,IAAAA,8BAAgB,EAACpB;oBAE9B;;wBAAMlB,KAAKI,IAAI,CAAC;;;oBAAhB;oBAEgB;;wBAAMK,IAAAA,iBAAO,EAACT,MAAM;;;oBAA9BoB,UAAU;oBACC;;wBAAMX,IAAAA,iBAAO,EAACT,MAAM;;;oBAA/BqB,WAAW;oBACXC,eAAeX,IAAAA,yBAAgB,EAACV,KAAKW,kBAAQ;yBAC/C,CAAA,AAACQ,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,YAAY,GAAnG;;;;oBACF,yFAAyF;oBACzF,iGAAiG;oBAC3FC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;oBACnH0C,QAAQC,KAAK,CAAC,AAAC,UAAgB,OAAPjB,QAAO;oBAC/BJ,SAAS;oBACT;;wBAAMD,GAAGuB,KAAK;;;oBAAd;oBACAC,IAAAA,mBAAU,EAACzC;oBACX;;wBAAOY,QAAQZ;;;oBAGjB;;wBAAMF,aAAaC,MAAMC;;;oBAAzB;oBAI8B;;wBAAMQ,IAAAA,iBAAO,EAACT,MAAM;;;oBAA5CwB,gBAAgBmB;yBAAQ,OAAA,2BAAA,kBAAA,OAA4C;;oBAC1E;;wBAAM3C,KAAKI,IAAI,CAAC,AAAC,yBAA8E,OAAtDwC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAItB,gBAAgB;;;oBAAtF;oBAE6B;;wBAAMuB,IAAAA,sBAAS,EAAC/C,MAAMC,KAAKA,IAAI2B,OAAO,EAAEoB,0BAAY;;;oBAApD,QAAA,eAArBvB,SAAqB,MAArBA,QAAQC,WAAa,MAAbA;oBAEhB;;wBAAO;4BAAER,IAAAA;4BAAIlB,MAAAA;4BAAMC,KAAAA;4BAAKc,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;oBAC1CT;yBACH,CAACE,QAAD;;;;oBAAS;;wBAAMD,GAAGuB,KAAK;;;oBAAd;;;oBACb,MAAMxB;;;;;;;IAEV;;AAIO,SAAenB,UAAUG,GAAmB;;YACgB,MAAzDiB,IAAIlB,MAAWiD,aAAalC,QAAQU,QAAQC,UAC9CwB;;;;oBAD2D;;wBAAMrC,QAAQZ;;;oBAAd,OAAA,eAAzDiB,KAAyD,KAAzDA,IAAIlB,OAAqD,KAArDA,MAAWiD,cAA0C,KAA/ChD,KAAkBc,SAA6B,KAA7BA,QAAQU,SAAqB,KAArBA,QAAQC,WAAa,KAAbA;oBAC9CwB,QAAQC,IAAAA,oBAAW,EAACjC,IAAIlB,MAAMiD,aAAaA,YAAYrB,OAAO;oBACpE,0FAA0F;oBAC1FwB,IAAAA,yBAAc,EAACpD,MAAMkD;oBACrB;;wBAAO;4BAAEA,OAAOA;4BAAOjD,KAAKgD;4BAAalC,QAAAA;4BAAQU,QAAAA;4BAAQC,UAAAA;wBAAS;;;;IACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","openTurso","tursoOpenDialect","ensureSchema","_handle","conn","cfg","ddl","feature","exec","CONTENT_FTS_DDL","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","db","tursoApi","SenseError","TURSO_PACKAGE","message","timeout","experimental","createConnection","filename","schemaVersion","reconcileDialect","tursoDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAGAC;eAAAA;;QAuESC;eAAAA;;QAfTC;eAAAA;;;uBAxEoB;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACO;2BACM;uBAClB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAI9B,4FAA4F;AAC5F,mFAAmF;AACnF,SAAeG,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;;YAGrE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,UAGN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBALX;;wBAAMH,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAaC,4BAAe;;;2BAA5B,6BAAA,QAAA;;;;oBAAMH,MAAN;oBAA8B;;wBAAMF,KAAKI,IAAI,CAACF;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBACL;;wBAAMF,KAAKI,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMJ,KAAKI,IAAI,CAAC;;;oBAAhB;oBACK,mCAAA,4BAAA;;;;;;;;;oBAAA,aAAiBE,IAAAA,wBAAc,EAACL;;;2BAAhC,8BAAA,SAAA;;;;oBAAME,UAAN;yBAGCA,CAAAA,QAAQI,IAAI,KAAK,OAAM,GAAvBJ;;;;oBACF;;wBAAMH,KAAKI,IAAI,CAAC,AAAC,wIAAkJ,OAAXI,mBAAU,EAAC;;;oBAAnK;oBACA;;;;;oBAEF;;wBAAML,QAAQM,MAAM,CAACT;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMU,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMe,IAAAA,iBAAO,EAACV,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMW,IAAAA,iBAAO,EAACX,MAAM,YAAYY,IAAAA,yBAAgB,EAACX,KAAKY,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,SAAeC,MAAMC,MAAgB;;;;;oBACnC;;wBAAMA,OAAOD,KAAK;;;oBAAlB;;;;;;IACF;;AAEA,SAAeE,sBAAsBjB,OAAiB,EAAEC,IAAgB,EAAEiB,EAAU;;;;;oBAClF;;wBAAMjB,KAAKI,IAAI,CAAC,AAAC,yBAA2B,OAAHa;;;oBAAzC;;;;;;IACF;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,OAGKC,KAKLC,IAKKD;;;;;;;;;;oBAXC;;wBAAME,IAAAA,kBAAQ;;;oBAAtBH,QAAQ;;;;;;oBACDC;oBACP,IAAIA,AAAG,YAAHA,KAAeG,oBAAU,GAAE,MAAMH;oBACrC,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,2BAAoD,OAA1BC,uBAAa,EAAC,cAAmC,OAAvB,AAACJ,IAAcK,OAAO,EAAC;;;;;;;;oBAOxH;;wBAAMN,MAAMH,OAAO,CAACC,QAAQ;4BAAES,SAAS;4BAAQC,YAAY;gCAAG;;wBAAgB;;;oBAFnF,4FAA4F;oBAC5F,uHAAuH;oBACvHN,KAAK;;;;;;oBACED;oBACP,MAAM,IAAIG,oBAAU,CAAC,4BAA4B,AAAC,gCAA0C,OAAXN,QAAO,MAA2B,OAAvB,AAACG,KAAcK,OAAO;;oBAEpH;;wBAAO;4BAAEZ,QAAQQ;4BAAIvB,MAAM8B,IAAAA,8BAAgB,EAACP;wBAAI;;;;IAClD;;AAGO,IAAM1B,mBAA0C;IACrDkC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,yBAAY;IAC9BhB,SAAAA;IACAJ,OAAAA;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGqB,UAAU,SAAVA,SAAWb;eAAQ,iEAAiEc,IAAI,CAACd,IAAIK,OAAO;;IACpG7B,cAAAA;IACAkB,uBAAAA;IACAqB,aAAa,SAAbA,YAActB,QAAQf;eAASqC,IAAAA,oBAAW,EAACtB,QAAQf;;AACrD;AAEO,SAAeJ,UAAUK,GAAmB;;;YACjD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKJ;;;IAC9B"}

package/dist/cjs/store/turso/reconcile.d.cts

@@ -1,3 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/cjs/store/turso/reconcile.d.ts

@@ -1,3 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/cjs/store/turso/reconcile.js

@@ -9,6 +9,12 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get CONTENT_FTS_DDL () {
+        return CONTENT_FTS_DDL;
+    },
+    get CONTENT_FTS_NAMES () {
+        return CONTENT_FTS_NAMES;
+    },
     get FTS_REBUILD_THRESHOLD () {
         return FTS_REBUILD_THRESHOLD;
     },
@@ -20,7 +26,6 @@ var _errorsts = require("../../errors.js");
 var _segmentts = require("../../text/segment.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
-var _opents = require("./open.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -146,8 +151,14 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
-// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
-// store/reconcile.ts. content is a plain table with "_ngram" sidecars for lexical.ts.
+var CONTENT_FTS_DDL = [
+    "CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')",
+    "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"
+];
+var CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 // '' when the field has no unspaced-script run (the common case), so the ngram index carries
 // nothing for it -- same "pay for nothing when absent" shape as sqlite's segmentField sidecars.
 function ngramSidecar(text) {
@@ -194,7 +205,7 @@ function reconcileContent(conn, touched, docs, delta) {
                         7,
                         8
                     ]);
-                    _iterator = _opents.CONTENT_FTS_NAMES[Symbol.iterator]();
+                    _iterator = CONTENT_FTS_NAMES[Symbol.iterator]();
                     _state.label = 2;
                 case 2:
                     if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
@@ -283,7 +294,7 @@ function reconcileContent(conn, touched, docs, delta) {
                         19,
                         20
                     ]);
-                    _iterator1 = _opents.CONTENT_FTS_DDL[Symbol.iterator]();
+                    _iterator1 = CONTENT_FTS_DDL[Symbol.iterator]();
                     _state.label = 14;
                 case 14:
                     if (!!(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done)) return [
@@ -338,6 +349,77 @@ function reconcileContent(conn, touched, docs, delta) {
         });
     })();
 }
+// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.
+function addColumns(conn, names) {
+    return _async_to_generator(function() {
+        var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, name, err;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                    _state.label = 1;
+                case 1:
+                    _state.trys.push([
+                        1,
+                        6,
+                        7,
+                        8
+                    ]);
+                    _iterator = names[Symbol.iterator]();
+                    _state.label = 2;
+                case 2:
+                    if (!!(_iteratorNormalCompletion = (_step = _iterator.next()).done)) return [
+                        3,
+                        5
+                    ];
+                    name = _step.value;
+                    return [
+                        4,
+                        conn.exec("ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name)))
+                    ];
+                case 3:
+                    _state.sent();
+                    _state.label = 4;
+                case 4:
+                    _iteratorNormalCompletion = true;
+                    return [
+                        3,
+                        2
+                    ];
+                case 5:
+                    return [
+                        3,
+                        8
+                    ];
+                case 6:
+                    err = _state.sent();
+                    _didIteratorError = true;
+                    _iteratorError = err;
+                    return [
+                        3,
+                        8
+                    ];
+                case 7:
+                    try {
+                        if (!_iteratorNormalCompletion && _iterator.return != null) {
+                            _iterator.return();
+                        }
+                    } finally{
+                        if (_didIteratorError) {
+                            throw _iteratorError;
+                        }
+                    }
+                    return [
+                        7
+                    ];
+                case 8:
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 var tursoDialect = {
     beginMode: function beginMode() {
         return _transactionts.BEGIN_WRITE;
@@ -347,9 +429,7 @@ var tursoDialect = {
             throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing turso's SELECT result-set column limit (").concat(MAX_FRONTMATTER_COLUMNS, "; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
         }
     },
-    columnTypeSuffix: function columnTypeSuffix() {
-        return '';
-    },
+    addColumns: addColumns,
     reconcileContent: reconcileContent,
     recordDuration: _sharedts.recordReconcileDuration
 };

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","CONTENT_FTS_NAMES","exec","runBatch","map","p","CONTENT_FTS_DDL","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAwBaA;eAAAA;;QA6BAC;eAAAA;;;wBArDc;yBAGI;wBACS;6BACZ;sBAEuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEnD,qFAAqF;AACrF,sFAAsF;AAEtF,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcO,yBAAiB;;;2BAA/B,6BAAA,QAAA;;;;oBAAMN,OAAN;oBAAiC;;wBAAMN,KAAKa,IAAI,CAAC,AAAC,wBAA4B,OAALP;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKc,QAAQ,CACjB,wCACAb,QAAQc,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEd,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKc,QAAQ,CAACtB,oBAAoBU,KAAKa,GAAG,CAACtB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAaY,uBAAe;;;2BAA5B,8BAAA,SAAA;;;;oBAAMV,MAAN;oBAA8B;;wBAAMP,KAAKa,IAAI,CAACN;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEO,IAAMpB,eAAiC;IAC5C+B,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ9B,yBAAyB;YACnC,MAAM,IAAI+B,oBAAU,CAClB,gBACA,AAAC,0BAA4F/B,OAAnE8B,OAAM,+DAAqF,OAAxB9B,yBAAwB;QAEzH;IACF;IACAgC,kBAAkB,SAAlBA;eAAwB;;IACxBxB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["CONTENT_FTS_DDL","CONTENT_FTS_NAMES","FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","exec","runBatch","map","p","addColumns","names","quoteIdent","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIAC;eAAAA;;QAcAC;eAAAA;;QAkCAC;eAAAA;;;wBAnEc;yBAGI;wBACqB;6BACxB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAUrB,IAAMH,kBAAkB;IAC7B;IACA;CACD;AACM,IAAMC,oBAAoB;IAAC;IAAe;CAAoB;AAErE,6FAA6F;AAC7F,gGAAgG;AAChG,SAASG,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcpB;;;2BAAd,6BAAA,QAAA;;;;oBAAMqB,OAAN;oBAAiC;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,wBAA4B,OAALN;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKa,QAAQ,CACjB,wCACAZ,QAAQa,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEb,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKa,QAAQ,CAACrB,oBAAoBU,KAAKY,GAAG,CAACrB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAarB;;;2BAAb,8BAAA,SAAA;;;;oBAAMuB,MAAN;oBAA8B;;wBAAMP,KAAKY,IAAI,CAACL;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEA,yFAAyF;AACzF,SAAeS,WAAWhB,IAAgB,EAAEiB,KAAe;;YACpD,2BAAA,mBAAA,gBAAA,WAAA,OAAMX;;;;oBAAN,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcW;;;2BAAd,6BAAA,QAAA;;;;oBAAMX,OAAN;oBAAqB;;wBAAMN,KAAKY,IAAI,CAAC,AAAC,sCAAsD,OAAjBM,IAAAA,oBAAU,EAACZ;;;oBAAjE;;;oBAArB;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACP;;AAEO,IAAMnB,eAAiC;IAC5CgC,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIgC,oBAAU,CAClB,gBACA,AAAC,0BAA4FhC,OAAnE+B,OAAM,+DAAqF,OAAxB/B,yBAAwB;QAEzH;IACF;IACAyB,YAAAA;IACAjB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/turso/store.d.cts

@@ -1,5 +1,4 @@
 import type { Database } from '@tursodatabase/database';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: Database, conn: Connection): Store;

package/dist/cjs/store/turso/store.d.ts

@@ -1,5 +1,4 @@
 import type { Database } from '@tursodatabase/database';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: Database, conn: Connection): Store;

package/dist/cjs/store/turso/store.js

@@ -17,13 +17,11 @@ _export(exports, {
     }
 });
 var _typests = require("../../embed/types.js");
-var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -368,7 +366,7 @@ var CAPABILITIES = new Set([
     'phrases',
     'vectors'
 ]);
-function createStore(db, conn, cfg, baseDir) {
+function createStore(db, conn) {
     return {
         name: 'turso',
         capabilities: CAPABILITIES,
@@ -428,16 +426,6 @@ function createStore(db, conn, cfg, baseDir) {
                 });
             })();
         },
-        reconcile: function reconcile() {
-            return _async_to_generator(function() {
-                return _ts_generator(this, function(_state) {
-                    return [
-                        2,
-                        (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.tursoDialect)
-                    ];
-                });
-            })();
-        },
         docs: {
             columns: function columns() {
                 return _async_to_generator(function() {