sensemaking 0.19.2 → 0.21.0

package/dist/esm/store/turso/open.js

@@ -1,34 +1,20 @@
-import { mkdirSync } from 'node:fs';
-import { join } from 'node:path';
-import { featureSignature, STATE_DIR } from '../../config/index.js';
-import { rekeyChunkText } from '../../embed/handoff.js';
+import { featureSignature } from '../../config/index.js';
 import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
-import { clearCache } from '../cache.js';
-import { reconcile } from '../reconcile.js';
+import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
 import { TURSO_PACKAGE, tursoApi } from './native.js';
-import { tursoDialect } from './reconcile.js';
+import { CONTENT_FTS_DDL, tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
 // separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
-export const SCHEMA_VERSION = '3';
-// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
-// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
-// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:
-// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
-export const CONTENT_FTS_DDL = [
-    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
-    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
-];
-export const CONTENT_FTS_NAMES = [
-    'content_fts',
-    'content_fts_ngram'
-];
-async function ensureSchema(conn, cfg) {
+export const SCHEMA_VERSION = '4';
+// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
+// extra native state to thread, unlike duckdb's separate instance/connection pair.
+async function ensureSchema(_handle, conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
     for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
@@ -46,12 +32,13 @@ async function ensureSchema(conn, cfg) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
-async function connect(cfg) {
-    const stateDir = join(cfg.baseDir, STATE_DIR);
-    mkdirSync(stateDir, {
-        recursive: true
-    });
-    const dbPath = join(stateDir, DB_FILENAME);
+async function close(handle) {
+    await handle.close();
+}
+async function setDerivedBusyTimeout(_handle, conn, ms) {
+    await conn.exec(`PRAGMA busy_timeout = ${ms}`);
+}
+async function connect(dbPath, _cfg) {
     // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this
     // optional dependency until a turso tree is actually opened. Installed on first use if missing.
     let turso;
@@ -74,57 +61,26 @@ async function connect(cfg) {
     } catch (err) {
         throw new SenseError('STORE_DEPENDENCY_MISSING', `store "turso" failed to open ${dbPath}: ${err.message}`);
     }
-    // A throw below must release this handle, or the leaked WAL makes the db file undeletable on
-    // Windows. closed guards the catch against double-closing.
-    let closed = false;
-    try {
-        var _ref;
-        const conn = createConnection(db);
-        await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
-        const version = await getMeta(conn, 'schema_version');
-        const features = await getMeta(conn, 'features');
-        const wantFeatures = featureSignature(cfg, FEATURES);
-        if (version !== null && version !== SCHEMA_VERSION || features !== null && features !== wantFeatures) {
-            // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched
-            // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.
-            const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';
-            console.error(`sense: ${reason}; rebuilding the index`);
-            closed = true;
-            await db.close();
-            clearCache(cfg);
-            return connect(cfg);
-        }
-        await ensureSchema(conn, cfg);
-        // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.
-        // Installed before reconcile() -- that call is the one that races a watcher's transaction.
-        const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
-        await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);
-        return {
-            db,
-            conn,
-            cfg,
-            dbPath,
-            parsed,
-            warnings
-        };
-    } catch (err) {
-        if (!closed) await db.close();
-        throw err;
-    }
-}
-// The turso store's open: connects (fully async, see connect() above), then wraps the resulting
-// connection in the Store interface.
-export async function openTurso(cfg) {
-    const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);
-    const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);
-    // reconcile ran before this object existed, so its chunk text is keyed by the connection.
-    rekeyChunkText(conn, store);
     return {
-        store: store,
-        cfg: resolvedCfg,
-        dbPath,
-        parsed,
-        warnings
+        handle: db,
+        conn: createConnection(db)
     };
 }
+// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.
+export const tursoOpenDialect = {
+    filename: DB_FILENAME,
+    schemaVersion: SCHEMA_VERSION,
+    reconcileDialect: tursoDialect,
+    connect,
+    close,
+    // "Locking error: Failed locking file ...", worded per platform: posix "File is locked by another
+    // process", Windows "another process has locked a portion of the file (os error 33)". Distinct from
+    // the write-time "database is locked" its connect-time `timeout` covers; that one never reaches here.
+    isLocked: (err)=>/File is locked by another process|locked a portion of the file/.test(err.message),
+    ensureSchema,
+    setDerivedBusyTimeout,
+    createStore: (handle, conn)=>createStore(handle, conn)
+};
+export async function openTurso(cfg) {
+    return openWithDialect(cfg, tursoOpenDialect);
+}

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '3';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:\n// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\ninterface ConnectResult {\n  db: Database;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below must release this handle, or the leaked WAL makes the db file undeletable on\n  // Windows. closed guards the catch against double-closing.\n  let closed = false;\n  try {\n    const conn = createConnection(db);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache is rebuilt rather than patched\n      // incrementally. No tokenize-only or embed-identity-adopted partial path: any mismatch rebuilds.\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      closed = true;\n      await db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    await conn.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, tursoDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) await db.close();\n    throw err;\n  }\n}\n\n// The turso store's open: connects (fully async, see connect() above), then wraps the resulting\n// connection in the Store interface.\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ensureSchema","conn","cfg","exec","ddl","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","turso","err","message","db","timeout","experimental","closed","version","features","wantFeatures","reason","console","error","close","recordedMaxMs","Number","Math","min","max","parsed","warnings","openTurso","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,YAAY,QAAQ,iBAAiB;AAC9C,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,+FAA+F;AAC/F,2FAA2F;AAC3F,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAON,gBAAiB,MAAMG,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBJ;IAC5F,IAAI,AAAC,MAAMR,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYrB,iBAAiBsB,KAAKhB;AACxG;AAWA,eAAesB,QAAQN,GAAmB;IACxC,MAAMO,WAAW9B,KAAKuB,IAAIQ,OAAO,EAAE7B;IACnCH,UAAU+B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASjC,KAAK8B,UAAUb;IAE9B,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIiB;IACJ,IAAI;QACFA,QAAQ,MAAMpB;IAChB,EAAE,OAAOqB,KAAK;QACZ,IAAIA,eAAe9B,YAAY,MAAM8B;QACrC,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACsB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAML,OAAO,CAACI,QAAQ;YAAEK,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI9B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACtH;IACA,6FAA6F;IAC7F,2DAA2D;IAC3D,IAAII,SAAS;IACb,IAAI;YAuB4B;QAtB9B,MAAMlB,OAAOV,iBAAiByB;QAE9B,MAAMf,KAAKE,IAAI,CAAC;QAEhB,MAAMiB,UAAU,MAAM/B,QAAQY,MAAM;QACpC,MAAMoB,WAAW,MAAMhC,QAAQY,MAAM;QACrC,MAAMqB,eAAe1C,iBAAiBsB,KAAKhB;QAC3C,IAAI,AAACkC,YAAY,QAAQA,YAAYvB,kBAAoBwB,aAAa,QAAQA,aAAaC,cAAe;YACxG,yFAAyF;YACzF,iGAAiG;YACjG,MAAMC,SAASH,YAAY,QAAQA,YAAYvB,iBAAiB,mDAAmD;YACnH2B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtDJ,SAAS;YACT,MAAMH,GAAGU,KAAK;YACdvC,WAAWe;YACX,OAAOM,QAAQN;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMyB,gBAAgBC,QAAQ,OAAA,MAAMvC,QAAQY,MAAM,iCAApB,kBAAA,OAA4C;QAC1E,MAAMA,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAEhG,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAM7C,UAAUa,MAAMC,KAAKA,IAAIQ,OAAO,EAAEhB;QAErE,OAAO;YAAEsB;YAAIf;YAAMC;YAAKU;YAAQoB;YAAQC;QAAS;IACnD,EAAE,OAAOnB,KAAK;QACZ,IAAI,CAACK,QAAQ,MAAMH,GAAGU,KAAK;QAC3B,MAAMZ;IACR;AACF;AAEA,gGAAgG;AAChG,qCAAqC;AACrC,OAAO,eAAeoB,UAAUhC,GAAmB;IACjD,MAAM,EAAEc,EAAE,EAAEf,IAAI,EAAEC,KAAKiC,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQN;IAC/E,MAAMkC,QAAQzC,YAAYqB,IAAIf,MAAMkC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F5B,eAAemB,MAAMmC;IACrB,OAAO;QAAEA,OAAOA;QAAOlC,KAAKiC;QAAavB;QAAQoB;QAAQC;IAAS;AACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWnB,eAAegB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEnB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMqB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYlB,iBAAiBmB,KAAKf;AACxG;AAEA,eAAeqB,MAAMC,MAAgB;IACnC,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAe/B,YAAY,MAAM+B;QACrC,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,wBAAwB,EAAEO,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAI/B,WAAW,4BAA4B,CAAC,6BAA6B,EAAE4B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOd,gBAAgBc,KAAKmB;AAC9B"}

package/dist/esm/store/turso/reconcile.d.ts

@@ -1,3 +1,5 @@
 import type { ReconcileDialect } from '../types.js';
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare const FTS_REBUILD_THRESHOLD = 250;
 export declare const tursoDialect: ReconcileDialect;

package/dist/esm/store/turso/reconcile.js

@@ -1,10 +1,21 @@
 import { SenseError } from '../../errors.js';
 import { hasUnspacedRun } from '../../text/segment.js';
-import { recordReconcileDuration } from '../shared.js';
+import { quoteIdent, recordReconcileDuration } from '../shared.js';
 import { BEGIN_WRITE } from '../transaction.js';
-import { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. content is a plain table with "_ngram" sidecars for lexical.ts.
+// The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
+// columns makes a bare substring match a whole word, defeating the prefix-query rejection.
+// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around
+// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
+export const CONTENT_FTS_DDL = [
+    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
+    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
+];
+export const CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 // '' when the field has no unspaced-script run (the common case), so the ngram index carries
 // nothing for it -- same "pay for nothing when absent" shape as sqlite's segmentField sidecars.
 function ngramSidecar(text) {
@@ -44,6 +55,10 @@ async function reconcileContent(conn, touched, docs, delta) {
     if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
     if (bulk) for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
 }
+// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.
+async function addColumns(conn, names) {
+    for (const name of names)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);
+}
 export const tursoDialect = {
     beginMode: ()=>BEGIN_WRITE,
     checkColumnLimit (count) {
@@ -51,7 +66,7 @@ export const tursoDialect = {
             throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
         }
     },
-    columnTypeSuffix: ()=>'',
+    addColumns,
     reconcileContent,
     recordDuration: recordReconcileDuration
 };

package/dist/esm/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","recordReconcileDuration","BEGIN_WRITE","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","tursoDialect","beginMode","checkColumnLimit","count","columnTypeSuffix","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,uBAAuB,QAAQ,eAAe;AACvD,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,SAASC,eAAe,EAAEC,iBAAiB,QAAQ,YAAY;AAE/D,qFAAqF;AACrF,sFAAsF;AAEtF,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAON,eAAeM,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,OAAO,MAAMC,eAAiC;IAC5CC,WAAW,IAAMhC;IACjBiC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ7B,yBAAyB;YACnC,MAAM,IAAIR,WACR,gBACA,CAAC,uBAAuB,EAAEqC,MAAM,2DAA2D,EAAE7B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA8B,kBAAkB,IAAM;IACxBrB;IACAsB,gBAAgBrC;AAClB,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { quoteIdent, recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// The ngram index is scoped to disjoint \"_ngram\" sidecar columns: a second index over the same\n// columns makes a bare substring match a whole word, defeating the prefix-query rejection.\n// The two FTS indexes, named here (not open.ts) because reconcile drops and rebuilds them around\n// a bulk load: Tantivy maintains them per inserted row, which is quadratic in what is already indexed.\nexport const CONTENT_FTS_DDL = [\n  `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,\n  `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`,\n] as const;\nexport const CONTENT_FTS_NAMES = ['content_fts', 'content_fts_ngram'] as const;\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\n// turso's ADD COLUMN is metadata-only, so a loop costs nothing extra over one statement.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  for (const name of names) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)}`);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  addColumns,\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["SenseError","hasUnspacedRun","quoteIdent","recordReconcileDuration","BEGIN_WRITE","CONTENT_FTS_DDL","CONTENT_FTS_NAMES","ngramSidecar","text","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","title","summary","search","relPath","reconcileContent","conn","touched","docs","delta","churn","reparsed","length","vanished","bulk","files","name","exec","runBatch","map","p","ddl","addColumns","names","tursoDialect","beginMode","checkColumnLimit","count","recordDuration"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,cAAc,QAAQ,wBAAwB;AACvD,SAASC,UAAU,EAAEC,uBAAuB,QAAQ,eAAe;AACnE,SAASC,WAAW,QAAQ,oBAAoB;AAGhD,qFAAqF;AACrF,sFAAsF;AAEtF,+FAA+F;AAC/F,2FAA2F;AAC3F,iGAAiG;AACjG,uGAAuG;AACvG,OAAO,MAAMC,kBAAkB;IAC7B,CAAC,qIAAqI,CAAC;IACvI,CAAC,gMAAgM,CAAC;CACnM,CAAU;AACX,OAAO,MAAMC,oBAAoB;IAAC;IAAe;CAAoB,CAAU;AAE/E,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOP,eAAeO,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,MAAMC,0BAA0B;AAEhC,4FAA4F;AAC5F,4EAA4E;AAC5E,OAAO,MAAMC,wBAAwB,IAAI;AAEzC,yFAAyF;AACzF,MAAMC,qBAAqB,CAAC,uHAAuH,CAAC;AAEpJ,SAASC,WAAWC,GAAc;IAChC,MAAM,EAAEC,KAAK,EAAEC,OAAO,EAAEP,IAAI,EAAE,GAAGK,IAAIG,MAAM;IAC3C,OAAO;QAACH,IAAII,OAAO;QAAEH;QAAOC;QAASP;QAAMD,aAAaO;QAAQP,aAAaQ;QAAUR,aAAaC;KAAM;AAC5G;AAEA,eAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;IAC3G,MAAMC,QAAQD,MAAME,QAAQ,CAACC,MAAM,GAAGH,MAAMI,QAAQ,CAACD,MAAM;IAE3D,6FAA6F;IAC7F,qDAAqD;IACrD,MAAME,OAAOL,MAAMM,KAAK,CAACH,MAAM,KAAK,KAAKF,QAAQb;IACjD,IAAIiB,MAAM,KAAK,MAAME,QAAQvB,kBAAmB,MAAMa,KAAKW,IAAI,CAAC,CAAC,qBAAqB,EAAED,MAAM;IAE9F,0FAA0F;IAC1F,8DAA8D;IAC9D,IAAIT,QAAQK,MAAM,GAAG,GACnB,MAAMN,KAAKY,QAAQ,CACjB,wCACAX,QAAQY,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIZ,KAAKI,MAAM,GAAG,GAAG,MAAMN,KAAKY,QAAQ,CAACpB,oBAAoBU,KAAKW,GAAG,CAACpB;IACtE,IAAIe,MAAM,KAAK,MAAMO,OAAO7B,gBAAiB,MAAMc,KAAKW,IAAI,CAACI;AAC/D;AAEA,yFAAyF;AACzF,eAAeC,WAAWhB,IAAgB,EAAEiB,KAAe;IACzD,KAAK,MAAMP,QAAQO,MAAO,MAAMjB,KAAKW,IAAI,CAAC,CAAC,mCAAmC,EAAE5B,WAAW2B,OAAO;AACpG;AAEA,OAAO,MAAMQ,eAAiC;IAC5CC,WAAW,IAAMlC;IACjBmC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQ/B,yBAAyB;YACnC,MAAM,IAAIT,WACR,gBACA,CAAC,uBAAuB,EAAEwC,MAAM,2DAA2D,EAAE/B,wBAAwB,gPAAgP,CAAC;QAE1W;IACF;IACA0B;IACAjB;IACAuB,gBAAgBtC;AAClB,EAAE"}

package/dist/esm/store/turso/store.d.ts

@@ -1,5 +1,4 @@
 import type { Database } from '@tursodatabase/database';
-import type { Config } from '../../config/index.js';
 import type { Capability, Connection, Store } from '../types.js';
 export declare const CAPABILITIES: ReadonlySet<Capability>;
-export declare function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store;
+export declare function createStore(db: Database, conn: Connection): Store;

package/dist/esm/store/turso/store.js

@@ -1,11 +1,9 @@
 import { STORE_DIMS } from '../../embed/types.js';
-import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
-import { tursoDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use
 // the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.
@@ -14,9 +12,9 @@ export const CAPABILITIES = new Set([
     'phrases',
     'vectors'
 ]);
-// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
+// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth
 // (see transaction.ts) is tracked against the same object everywhere.
-export function createStore(db, conn, cfg, baseDir) {
+export function createStore(db, conn) {
     return {
         name: 'turso',
         capabilities: CAPABILITIES,
@@ -32,9 +30,6 @@ export function createStore(db, conn, cfg, baseDir) {
         async transaction (fn) {
             return withTransaction(conn, fn);
         },
-        async reconcile () {
-            return reconcile(conn, cfg, baseDir, tursoDialect);
-        },
         docs: {
             async columns () {
                 return [

package/dist/esm/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, tursoDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","tursoDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,YAAY,QAAQ,iBAAiB;AAC9C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8FAA8F;AAC9F,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLC,MAAM;QACNC,cAAcR;QACd,MAAMS,MAAKC,GAAW;YACpB,MAAMN,KAAKK,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAON,KAAKO,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMT,KAAKQ,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBa,MAAMW;QAC/B;QACA,MAAM1B;YACJ,OAAOA,UAAUe,MAAMC,KAAKC,SAASV;QACvC;QACAoB,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM3B,WAAWc;iBAAO;YACtC;YACAV,YAAY,CAACuB,SAASC,aAAexB,WAAWU,MAAMa,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAAS3B,aAAaS,MAAMiB,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM/B,YAAYW;YAC3BqB,cAAc,CAACC,OAAS3B,iBAAiBK,MAAMhB,YAAYsC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYlC,eAAeO,MAAMwB,IAAIxC,YAAY0C,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAASxB,YAAYM,MAAMhB,YAAY6C,MAAMX;YAC7DY,WAAW,CAACD,OAASzC,aAAaY,MAAM6B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAMjC,GAAGQ,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMrC,GAAGQ,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMzC,GAAGyC,KAAK;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,4FAA4F;AAC5F,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOrB,gBAAgBY,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMxB,WAAWa;iBAAO;YACtC;YACAT,YAAY,CAACoB,SAASC,aAAerB,WAAWS,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAASxB,aAAaQ,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM5B,YAAYU;YAC3BmB,cAAc,CAACC,OAASzB,iBAAiBK,MAAMd,YAAYkC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,MAAMsB,IAAIpC,YAAYsC,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAAStB,YAAYM,MAAMd,YAAYyC,MAAMX;YAC7DY,WAAW,CAACD,OAAStC,aAAaW,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMvC,GAAGuC,KAAK;QAChB;IACF;AACF"}

package/dist/esm/store/types.d.ts

@@ -1,4 +1,4 @@
-import type { Config, StoreName } from '../config/index.js';
+import type { Config, ResolvedConfig, StoreName } from '../config/index.js';
 import type { ReconcileDelta } from '../features/types.js';
 import type { ParsedDoc } from '../scan/index.js';
 export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
@@ -25,10 +25,24 @@ export interface Connection {
 export interface ReconcileDialect {
     beginMode(): string;
     checkColumnLimit(count: number): void;
-    columnTypeSuffix(): string;
+    addColumns(conn: Connection, names: string[]): Promise<void>;
     reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
     recordDuration?(conn: Connection, ms: number): Promise<void>;
 }
+export interface OpenDialect<Handle> {
+    filename: string;
+    schemaVersion: string;
+    reconcileDialect: ReconcileDialect;
+    connect(dbPath: string, cfg: ResolvedConfig): Promise<{
+        handle: Handle;
+        conn: Connection;
+    }>;
+    close(handle: Handle): Promise<void>;
+    isLocked?(err: Error): boolean;
+    ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;
+    setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;
+    createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;
+}
 export interface FieldStat {
     field: string;
     coverage: number;
@@ -96,10 +110,6 @@ export interface Store {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
     transaction<T>(fn: () => Promise<T>): Promise<T>;
-    reconcile(): Promise<{
-        parsed: number;
-        warnings: string[];
-    }>;
     docs: DocumentStore;
     lexical: LexicalIndex;
     vectors: VectorStore;

package/dist/esm/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Type suffix for ALTER TABLE ADD COLUMN: '' for sqlite/turso, ' VARIANT' for duckdb.\n  columnTypeSuffix(): string;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,CAAU"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, ResolvedConfig, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Adds `names` to frontmatter, already filtered to columns this connection doesn't have yet.\n  // sqlite/turso loop (their ADD COLUMN is metadata-only); duckdb issues one statement per call.\n  addColumns(conn: Connection, names: string[]): Promise<void>;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\n// One open algorithm (src/store/open.ts), parameterised per engine. `Handle` is whatever this\n// store needs to close the connection and construct its Store (sqlite: {db}; duckdb:\n// {instance, duckdb}; turso: db) -- opaque to the shared orchestration, threaded through unchanged.\nexport interface OpenDialect<Handle> {\n  // Cache filename under STATE_DIR, e.g. 'cache.db'.\n  filename: string;\n  // Cache shape version, independent of the config's own `version`; bumping it rebuilds an existing tree.\n  schemaVersion: string;\n  reconcileDialect: ReconcileDialect;\n  // Opens the physical connection and applies pragmas due before any SQL runs (sqlite: busy_timeout\n  // + WAL; turso: connect-time timeout; duckdb: none).\n  connect(dbPath: string, cfg: ResolvedConfig): Promise<{ handle: Handle; conn: Connection }>;\n  // Releases the handle, for both the rebuild-and-reopen branch and error cleanup on this attempt.\n  close(handle: Handle): Promise<void>;\n  // True when connect() failed because another process holds the cache file, which the orchestration\n  // retries. Each engine words it differently, so the match is the dialect's own (`native-not-emulated`).\n  // Absent for sqlite, whose file lock is shared and whose concurrent-open failure is a different defect.\n  // Matching on message text is forced, not chosen: measured 2026-09-02, duckdb throws a plain Error\n  // whose only own properties are stack and message, and turso sets code to the constant\n  // 'GenericFailure' on every failure alike with rawCode undefined. Neither exposes anything a\n  // predicate could switch on, so each dialect pins its engine's wordings and unit-tests them.\n  isLocked?(err: Error): boolean;\n  // Schema DDL beyond frontmatter/preset_files/meta (content table, feature hooks); sole owner of\n  // whether it wraps itself in a write transaction (sqlite: yes, guards a cold-open ALTER race; duckdb/turso: no).\n  ensureSchema(handle: Handle, conn: Connection, cfg: Config): Promise<void>;\n  // Installs the derived busy_timeout PRAGMA right before reconcile (sqlite/turso); absent for\n  // duckdb, which has no such PRAGMA.\n  setDerivedBusyTimeout?(handle: Handle, conn: Connection, ms: number): Promise<void>;\n  createStore(handle: Handle, conn: Connection, cfg: ResolvedConfig): Store;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;AAM5F,6FAA6F;AAC7F,mGAAmG;AACnG,mGAAmG;AACnG,kGAAkG;AAClG,OAAO,MAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB,CAAU"}

package/dist/esm/text/strip.js

@@ -1,10 +1,9 @@
-import { extractText, parse } from '../chunk/index.js';
+import { extractTexts, parse } from '../chunk/index.js';
 import { normalizeText } from '../scan/frontmatter.js';
-// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses
+// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses
 // (src/chunk) -- words and vectors read one definition of "the prose of this note".
 export function textFromBlocks(blocks) {
-    const text = blocks.map((block)=>extractText(block.node)).filter((s)=>s.length > 0).join('\n');
-    return normalizeText(text);
+    return normalizeText(extractTexts(blocks.map((block)=>block.node)));
 }
 // Thin parse + delegate, for callers with no blocks of their own already.
 export function stripText(value) {

package/dist/esm/text/strip.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractText, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractText the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  const text = blocks\n    .map((block) => extractText(block.node))\n    .filter((s) => s.length > 0)\n    .join('\\n');\n  return normalizeText(text);\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractText","parse","normalizeText","textFromBlocks","blocks","text","map","block","node","filter","s","length","join","stripText","value"],"mappings":"AACA,SAASA,WAAW,EAAEC,KAAK,QAAQ,oBAAoB;AACvD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,4FAA4F;AAC5F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,MAAMC,OAAOD,OACVE,GAAG,CAAC,CAACC,QAAUP,YAAYO,MAAMC,IAAI,GACrCC,MAAM,CAAC,CAACC,IAAMA,EAAEC,MAAM,GAAG,GACzBC,IAAI,CAAC;IACR,OAAOV,cAAcG;AACvB;AAEA,0EAA0E;AAC1E,OAAO,SAASQ,UAAUC,KAAa;IACrC,OAAOX,eAAeF,MAAMa;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks.map((block) => block.node)));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","map","block","node","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI,OAAOC,GAAG,CAAC,CAACC,QAAUA,MAAMC,IAAI;AACpE;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAON,eAAeF,MAAMQ;AAC9B"}

package/dist/esm/watch.js

@@ -2,7 +2,7 @@ import { watch as fsWatch } from 'node:fs';
 import { STATE_DIR } from './config/index.js';
 import { SenseError } from './errors.js';
 import { guardedTick } from './lib/guarded-tick.js';
-import { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.js';
+import { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.js';
 // Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.
 const DEBOUNCE_MS = 200;
 const HEARTBEAT_INTERVAL_MS = 5000;
@@ -14,12 +14,13 @@ export async function runWatch(cfg, opts = {}) {
     const onEvent = (_opts_onEvent = opts.onEvent) !== null && _opts_onEvent !== void 0 ? _opts_onEvent : ()=>{};
     const debounceMs = (_opts_debounceMs = opts.debounceMs) !== null && _opts_debounceMs !== void 0 ? _opts_debounceMs : DEBOUNCE_MS;
     const heartbeatIntervalMs = (_opts_heartbeatIntervalMs = opts.heartbeatIntervalMs) !== null && _opts_heartbeatIntervalMs !== void 0 ? _opts_heartbeatIntervalMs : HEARTBEAT_INTERVAL_MS;
-    const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);
+    const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);
     const baseDir = cfg.baseDir;
     const existingHeartbeat = await getMeta(store, 'watch_heartbeat');
     if (existingHeartbeat && !opts.force) {
         const age = Date.now() - Date.parse(existingHeartbeat);
         if (age >= 0 && age < STALE_HEARTBEAT_MS) {
+            await builder.close();
             await store.close();
             throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);
         }
@@ -53,7 +54,7 @@ export async function runWatch(cfg, opts = {}) {
             debounceTimer = null;
             inFlight = (async ()=>{
                 try {
-                    const { parsed, warnings } = await store.reconcile();
+                    const { parsed, warnings } = await builder.build();
                     onEvent({
                         type: 'reconciled',
                         parsed,
@@ -95,6 +96,7 @@ export async function runWatch(cfg, opts = {}) {
             if (debounceTimer) clearTimeout(debounceTimer);
             watcher.close();
             await inFlight;
+            await builder.close();
             await setMeta(store, 'watch_heartbeat', null);
             await setMeta(store, 'watch_pid', null);
             await store.close();

package/dist/esm/watch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStore, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStore(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await store.reconcile();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","reconcile","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,SAAS,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAElG,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMjB,UAAUO;IAC5F,MAAMW,UAAUX,IAAIW,OAAO;IAE3B,MAAMC,oBAAoB,MAAMpB,QAAQa,OAAO;IAC/C,IAAIO,qBAAqB,CAACX,KAAKY,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMhB,oBAAoB;YACxC,MAAMO,MAAMa,KAAK;YACjB,MAAM,IAAI7B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE8B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAZ,QAAQ;QAAEmB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDR,QAAQ;YAAEmB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMhC,SAASc;YAAQE,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM9B,QAAQU,OAAO,mBAAmB,IAAIU,OAAOW,WAAW;QAC9D,MAAM/B,QAAQU,OAAO,aAAasB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,MAAM8B,SAAS;oBAClDjC,QAAQ;wBAAEmB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMhC,SAASc;wBAAQE;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZlC,QAAQ;wBAAEmB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG5B;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAMmC,UAAUnD,QAAQwB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACtD,YAAY;QACpE4C;IACF;IACA,MAAMW,iBAAiBC,YACrBtD,YAAYmC,gBAAgB,IAAMD,WAClCpB;IAGF,OAAO,IAAIyC,QAAc,CAACC;YAoBpB7C,cACCA;QApBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM8C,WAAW;gBAKf9C;YAJA,IAAIuB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaiD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAMpC,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMa,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI9C,eAAAA,KAAKgD,MAAM,cAAXhD,mCAAAA,aAAaoD,OAAO,EAAEN;cACrB9C,gBAAAA,KAAKgD,MAAM,cAAXhD,oCAAAA,cAAaqD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/watch.ts"],"sourcesContent":["import { watch as fsWatch } from 'node:fs';\nimport type { ResolvedConfig } from './config/index.ts';\nimport { STATE_DIR } from './config/index.ts';\nimport { SenseError } from './errors.ts';\nimport { guardedTick } from './lib/guarded-tick.ts';\nimport { docCount, getMeta, openStoreForWatch, requireWatchConcurrency, setMeta } from './store/index.ts';\n\n// Watch is a cache pre-warmer, not a correctness mechanism: open() always reconciles anyway, so any fs event just triggers a debounced full reconcile.\nconst DEBOUNCE_MS = 200;\nconst HEARTBEAT_INTERVAL_MS = 5000;\nconst STALE_HEARTBEAT_MS = 15000;\n\nexport type WatchEvent = { type: 'started'; baseDir: string; dbPath: string } | { type: 'reconciled'; parsed: number; total: number; warnings: string[] } | { type: 'reconcile-error'; message: string };\n\nexport interface WatchOptions {\n  force?: boolean;\n  onEvent?: (event: WatchEvent) => void;\n  // Aborting runs the same shutdown path as SIGINT/SIGTERM.\n  signal?: AbortSignal;\n  debounceMs?: number;\n  heartbeatIntervalMs?: number;\n}\n\n// Runs in the foreground until SIGINT/SIGTERM/signal abort. Throws WATCH_ACTIVE if another watcher's heartbeat is still fresh and --force wasn't given.\nexport async function runWatch(cfg: ResolvedConfig, opts: WatchOptions = {}): Promise<void> {\n  requireWatchConcurrency(cfg);\n  const onEvent = opts.onEvent ?? (() => {});\n  const debounceMs = opts.debounceMs ?? DEBOUNCE_MS;\n  const heartbeatIntervalMs = opts.heartbeatIntervalMs ?? HEARTBEAT_INTERVAL_MS;\n  const { store, builder, dbPath, warnings: initialWarnings, parsed: initialParsed } = await openStoreForWatch(cfg);\n  const baseDir = cfg.baseDir;\n\n  const existingHeartbeat = await getMeta(store, 'watch_heartbeat');\n  if (existingHeartbeat && !opts.force) {\n    const age = Date.now() - Date.parse(existingHeartbeat);\n    if (age >= 0 && age < STALE_HEARTBEAT_MS) {\n      await builder.close();\n      await store.close();\n      throw new SenseError('WATCH_ACTIVE', `another watcher appears active (heartbeat ${Math.round(age / 1000)}s ago); use --force to override`);\n    }\n  }\n\n  onEvent({ type: 'started', baseDir, dbPath });\n  if (initialWarnings.length > 0 || initialParsed > 0) {\n    onEvent({ type: 'reconciled', parsed: initialParsed, total: await docCount(store), warnings: initialWarnings });\n  }\n\n  let stopping = false;\n  const touchHeartbeat = async () => {\n    await setMeta(store, 'watch_heartbeat', new Date().toISOString());\n    await setMeta(store, 'watch_pid', String(process.pid));\n  };\n  await touchHeartbeat();\n\n  let debounceTimer: NodeJS.Timeout | null = null;\n  // A reconcile that has already started owns the store, and on a bulk reparse a live worker\n  // pool as well. Shutdown waits on this rather than closing the connection underneath it.\n  let inFlight: Promise<void> | null = null;\n  const scheduleReconcile = () => {\n    if (debounceTimer) clearTimeout(debounceTimer);\n    debounceTimer = setTimeout(() => {\n      debounceTimer = null;\n      inFlight = (async () => {\n        try {\n          const { parsed, warnings } = await builder.build();\n          onEvent({ type: 'reconciled', parsed, total: await docCount(store), warnings });\n        } catch (err) {\n          onEvent({ type: 'reconcile-error', message: (err as Error).message });\n        } finally {\n          inFlight = null;\n        }\n      })();\n    }, debounceMs);\n  };\n\n  // Ignore our own state dir, or the heartbeat write would retrigger itself forever. An event with\n  // an unresolvable filename (null, which fs.watch delivers under load) reconciles: parsing nothing costs less than missing a real edit.\n  const watcher = fsWatch(baseDir, { recursive: true }, (_event, filename) => {\n    if (typeof filename === 'string' && filename.startsWith(STATE_DIR)) return;\n    scheduleReconcile();\n  });\n  const heartbeatTimer = setInterval(\n    guardedTick(touchHeartbeat, () => stopping),\n    heartbeatIntervalMs\n  );\n\n  return new Promise<void>((resolveShutdown) => {\n    // SIGINT/SIGTERM and an aborted signal all run this same path exactly once; each is\n    // unregistered here too so a second runWatch call in the same process starts clean.\n    const shutdown = async () => {\n      if (stopping) return;\n      stopping = true;\n      process.off('SIGINT', shutdown);\n      process.off('SIGTERM', shutdown);\n      opts.signal?.removeEventListener('abort', shutdown);\n      clearInterval(heartbeatTimer);\n      if (debounceTimer) clearTimeout(debounceTimer);\n      watcher.close();\n      await inFlight;\n      await builder.close();\n      await setMeta(store, 'watch_heartbeat', null);\n      await setMeta(store, 'watch_pid', null);\n      await store.close();\n      resolveShutdown();\n    };\n    process.once('SIGINT', shutdown);\n    process.once('SIGTERM', shutdown);\n    if (opts.signal?.aborted) shutdown();\n    else opts.signal?.addEventListener('abort', shutdown, { once: true });\n  });\n}\n"],"names":["watch","fsWatch","STATE_DIR","SenseError","guardedTick","docCount","getMeta","openStoreForWatch","requireWatchConcurrency","setMeta","DEBOUNCE_MS","HEARTBEAT_INTERVAL_MS","STALE_HEARTBEAT_MS","runWatch","cfg","opts","onEvent","debounceMs","heartbeatIntervalMs","store","builder","dbPath","warnings","initialWarnings","parsed","initialParsed","baseDir","existingHeartbeat","force","age","Date","now","parse","close","Math","round","type","length","total","stopping","touchHeartbeat","toISOString","String","process","pid","debounceTimer","inFlight","scheduleReconcile","clearTimeout","setTimeout","build","err","message","watcher","recursive","_event","filename","startsWith","heartbeatTimer","setInterval","Promise","resolveShutdown","shutdown","off","signal","removeEventListener","clearInterval","once","aborted","addEventListener"],"mappings":"AAAA,SAASA,SAASC,OAAO,QAAQ,UAAU;AAE3C,SAASC,SAAS,QAAQ,oBAAoB;AAC9C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,WAAW,QAAQ,wBAAwB;AACpD,SAASC,QAAQ,EAAEC,OAAO,EAAEC,iBAAiB,EAAEC,uBAAuB,EAAEC,OAAO,QAAQ,mBAAmB;AAE1G,uJAAuJ;AACvJ,MAAMC,cAAc;AACpB,MAAMC,wBAAwB;AAC9B,MAAMC,qBAAqB;AAa3B,wJAAwJ;AACxJ,OAAO,eAAeC,SAASC,GAAmB,EAAEC,OAAqB,CAAC,CAAC;QAEzDA,eACGA,kBACSA;IAH5BP,wBAAwBM;IACxB,MAAME,WAAUD,gBAAAA,KAAKC,OAAO,cAAZD,2BAAAA,gBAAiB,KAAO;IACxC,MAAME,cAAaF,mBAAAA,KAAKE,UAAU,cAAfF,8BAAAA,mBAAmBL;IACtC,MAAMQ,uBAAsBH,4BAAAA,KAAKG,mBAAmB,cAAxBH,uCAAAA,4BAA4BJ;IACxD,MAAM,EAAEQ,KAAK,EAAEC,OAAO,EAAEC,MAAM,EAAEC,UAAUC,eAAe,EAAEC,QAAQC,aAAa,EAAE,GAAG,MAAMlB,kBAAkBO;IAC7G,MAAMY,UAAUZ,IAAIY,OAAO;IAE3B,MAAMC,oBAAoB,MAAMrB,QAAQa,OAAO;IAC/C,IAAIQ,qBAAqB,CAACZ,KAAKa,KAAK,EAAE;QACpC,MAAMC,MAAMC,KAAKC,GAAG,KAAKD,KAAKE,KAAK,CAACL;QACpC,IAAIE,OAAO,KAAKA,MAAMjB,oBAAoB;YACxC,MAAMQ,QAAQa,KAAK;YACnB,MAAMd,MAAMc,KAAK;YACjB,MAAM,IAAI9B,WAAW,gBAAgB,CAAC,0CAA0C,EAAE+B,KAAKC,KAAK,CAACN,MAAM,MAAM,+BAA+B,CAAC;QAC3I;IACF;IAEAb,QAAQ;QAAEoB,MAAM;QAAWV;QAASL;IAAO;IAC3C,IAAIE,gBAAgBc,MAAM,GAAG,KAAKZ,gBAAgB,GAAG;QACnDT,QAAQ;YAAEoB,MAAM;YAAcZ,QAAQC;YAAea,OAAO,MAAMjC,SAASc;YAAQG,UAAUC;QAAgB;IAC/G;IAEA,IAAIgB,WAAW;IACf,MAAMC,iBAAiB;QACrB,MAAM/B,QAAQU,OAAO,mBAAmB,IAAIW,OAAOW,WAAW;QAC9D,MAAMhC,QAAQU,OAAO,aAAauB,OAAOC,QAAQC,GAAG;IACtD;IACA,MAAMJ;IAEN,IAAIK,gBAAuC;IAC3C,2FAA2F;IAC3F,yFAAyF;IACzF,IAAIC,WAAiC;IACrC,MAAMC,oBAAoB;QACxB,IAAIF,eAAeG,aAAaH;QAChCA,gBAAgBI,WAAW;YACzBJ,gBAAgB;YAChBC,WAAW,AAAC,CAAA;gBACV,IAAI;oBACF,MAAM,EAAEtB,MAAM,EAAEF,QAAQ,EAAE,GAAG,MAAMF,QAAQ8B,KAAK;oBAChDlC,QAAQ;wBAAEoB,MAAM;wBAAcZ;wBAAQc,OAAO,MAAMjC,SAASc;wBAAQG;oBAAS;gBAC/E,EAAE,OAAO6B,KAAK;oBACZnC,QAAQ;wBAAEoB,MAAM;wBAAmBgB,SAAS,AAACD,IAAcC,OAAO;oBAAC;gBACrE,SAAU;oBACRN,WAAW;gBACb;YACF,CAAA;QACF,GAAG7B;IACL;IAEA,iGAAiG;IACjG,uIAAuI;IACvI,MAAMoC,UAAUpD,QAAQyB,SAAS;QAAE4B,WAAW;IAAK,GAAG,CAACC,QAAQC;QAC7D,IAAI,OAAOA,aAAa,YAAYA,SAASC,UAAU,CAACvD,YAAY;QACpE6C;IACF;IACA,MAAMW,iBAAiBC,YACrBvD,YAAYoC,gBAAgB,IAAMD,WAClCrB;IAGF,OAAO,IAAI0C,QAAc,CAACC;YAqBpB9C,cACCA;QArBL,oFAAoF;QACpF,oFAAoF;QACpF,MAAM+C,WAAW;gBAKf/C;YAJA,IAAIwB,UAAU;YACdA,WAAW;YACXI,QAAQoB,GAAG,CAAC,UAAUD;YACtBnB,QAAQoB,GAAG,CAAC,WAAWD;aACvB/C,eAAAA,KAAKiD,MAAM,cAAXjD,mCAAAA,aAAakD,mBAAmB,CAAC,SAASH;YAC1CI,cAAcR;YACd,IAAIb,eAAeG,aAAaH;YAChCQ,QAAQpB,KAAK;YACb,MAAMa;YACN,MAAM1B,QAAQa,KAAK;YACnB,MAAMxB,QAAQU,OAAO,mBAAmB;YACxC,MAAMV,QAAQU,OAAO,aAAa;YAClC,MAAMA,MAAMc,KAAK;YACjB4B;QACF;QACAlB,QAAQwB,IAAI,CAAC,UAAUL;QACvBnB,QAAQwB,IAAI,CAAC,WAAWL;QACxB,KAAI/C,eAAAA,KAAKiD,MAAM,cAAXjD,mCAAAA,aAAaqD,OAAO,EAAEN;cACrB/C,gBAAAA,KAAKiD,MAAM,cAAXjD,oCAAAA,cAAasD,gBAAgB,CAAC,SAASP,UAAU;YAAEK,MAAM;QAAK;IACrE;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.19.2",
+  "version": "0.21.0",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -72,9 +72,10 @@
     "schema.json"
   ],
   "scripts": {
+    "benchmark": "node benchmark/gate.mjs",
     "build": "tsds build",
     "format": "tsds format",
-    "prepublishOnly": "tsds validate",
+    "prepublishOnly": "node scripts/prepublish.ts && tsds validate",
     "test": "tsds test:node --no-timeouts",
     "test:engines": "nvu engines tsds test:node --no-timeouts",
     "version": "tsds version"
@@ -82,7 +83,7 @@
   "dependencies": {
     "@huggingface/tokenizers": "^0.1.3",
     "franc-min": "^6.2.0",
-    "install-module-linked": "^1.6.5",
+    "install-module-linked": "^2.0.0",
     "mdast-util-from-markdown": "^2.0.3",
     "mdast-util-gfm-autolink-literal": "^2.0.0",
     "mdast-util-gfm-footnote": "^2.0.0",