sensemaking 0.22.2 → 0.23.1

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '19';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DatabaseSync","featureSignature","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","schema","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","handle","close","ensureSchema","_handle","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;IAC7D,MAAMH,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMA,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,KAAK,MAAMG,WAAWnB,eAAekB,KAAM,MAAMC,QAAQC,MAAM,CAACL;IAChE,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYhB,iBAAiBmB,KAAKjB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASoB,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGN,IAAI,CAAC;YACR;QACF,EAAE,OAAOU,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,MAAMb,KAAK,IAAIxB,aAAaoC;IAC5B,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFZ,GAAGN,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEK,cAAcC;QACdb,kBAAkBa;QAClB,MAAMP,OAAOR,iBAAiBe;QAC9B,OAAO;YAAEc,QAAQ;gBAAEd;YAAG;YAAGP;QAAK;IAChC,EAAE,OAAOW,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGe,KAAK;QACR,MAAMX;IACR;AACF;AAEA,eAAeW,MAAMD,MAAoB;IACvCA,OAAOd,EAAE,CAACe,KAAK;AACjB;AAEA,eAAeC,aAAaC,OAAqB,EAAExB,IAAgB,EAAEG,GAAW;IAC9E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgBS,MAAM,IAAME,mBAAmBF,MAAMG,MAAMb;AACnE;AAEA,eAAemC,sBAAsBJ,MAAoB,EAAEK,KAAiB,EAAEC,EAAU;IACtFN,OAAOd,EAAE,CAACN,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUjC;IACVkC,eAAejC;IACfkC,kBAAkBtC;IAClByB;IACAI;IACAC;IACAE;IACA9B,aAAa,CAAC0B,QAAQrB,OAASL,YAAY0B,OAAOd,EAAE,EAAEP;AACxD,EAAE;AAEF,OAAO,eAAegC,WAAW7B,GAAmB;IAClD,OAAOhB,gBAAgBgB,KAAKyB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '20';\n\nexport type { OpenResult };\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; the `_seg` sidecars (always populated) cover that case.\nconst TOKENIZE = 'porter unicode61';\n\ninterface SqliteHandle {\n  db: DatabaseSync;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${TOKENIZE}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchemaTables(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await createContentTable(conn);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: SqliteHandle; conn: Connection }> {\n  const db = new DatabaseSync(dbPath);\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db);\n    const conn = createConnection(db);\n    return { handle: { db }, conn };\n  } catch (err) {\n    // A throw here must release this handle, or on Windows the leaked WAL db is undeletable and\n    // scratch cleanup fails with EPERM/EBUSY.\n    db.close();\n    throw err;\n  }\n}\n\nasync function close(handle: SqliteHandle): Promise<void> {\n  handle.db.close();\n}\n\nasync function ensureSchema(_handle: SqliteHandle, conn: Connection, cfg: Config): Promise<void> {\n  // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n  // here both see it missing and the second ALTER fails with a duplicate column.\n  await withTransaction(conn, () => ensureSchemaTables(conn, cfg), BEGIN_WRITE);\n}\n\nasync function setDerivedBusyTimeout(handle: SqliteHandle, _conn: Connection, ms: number): Promise<void> {\n  handle.db.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const sqliteOpenDialect: OpenDialect<SqliteHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: sqliteDialect,\n  connect,\n  close,\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle.db, conn),\n};\n\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, sqliteOpenDialect);\n}\n"],"names":["DatabaseSync","featureSignature","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","TOKENIZE","createContentTable","conn","exec","ensureSchemaTables","cfg","feature","schema","setJournalWal","db","deadline","Date","now","err","test","message","Atomics","wait","Int32Array","SharedArrayBuffer","connect","dbPath","_cfg","handle","close","ensureSchema","_handle","setDerivedBusyTimeout","_conn","ms","sqliteOpenDialect","filename","schemaVersion","reconcileDialect","openSqlite"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,YAAY,QAAQ,cAAc;AAE3C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAInC,4FAA4F;AAC5F,+FAA+F;AAC/F,MAAMC,WAAW;AAMjB,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeC,mBAAmBC,IAAgB;IAChD,MAAMA,KAAKC,IAAI,CAAC,CAAC,0IAA0I,EAAEH,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeI,mBAAmBF,IAAgB,EAAEG,GAAW;IAC7D,MAAMH,KAAKC,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,mBAAmBC;IACzB,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMA,KAAKC,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMD,KAAKC,IAAI,CAAC;IAChB,KAAK,MAAMG,WAAWnB,eAAekB,KAAM,MAAMC,QAAQC,MAAM,CAACL;IAChE,IAAI,AAAC,MAAMZ,QAAQY,MAAM,sBAAuB,MAAM,MAAMX,QAAQW,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMT,QAAQY,MAAM,gBAAiB,MAAM,MAAMX,QAAQW,MAAM,YAAYhB,iBAAiBmB,KAAKjB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASoB,cAAcC,EAAgB;IACrC,MAAMC,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFH,GAAGN,IAAI,CAAC;YACR;QACF,EAAE,OAAOU,KAAK;YACZ,IAAIF,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BI,IAAI,CAAC,AAACD,IAAcE,OAAO,GAAG,MAAMF;YAC9FG,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,MAAMb,KAAK,IAAIxB,aAAaoC;IAC5B,IAAI;QACF,6FAA6F;QAC7F,yFAAyF;QACzFZ,GAAGN,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEK,cAAcC;QACdb,kBAAkBa;QAClB,MAAMP,OAAOR,iBAAiBe;QAC9B,OAAO;YAAEc,QAAQ;gBAAEd;YAAG;YAAGP;QAAK;IAChC,EAAE,OAAOW,KAAK;QACZ,4FAA4F;QAC5F,0CAA0C;QAC1CJ,GAAGe,KAAK;QACR,MAAMX;IACR;AACF;AAEA,eAAeW,MAAMD,MAAoB;IACvCA,OAAOd,EAAE,CAACe,KAAK;AACjB;AAEA,eAAeC,aAAaC,OAAqB,EAAExB,IAAgB,EAAEG,GAAW;IAC9E,+FAA+F;IAC/F,+EAA+E;IAC/E,MAAMZ,gBAAgBS,MAAM,IAAME,mBAAmBF,MAAMG,MAAMb;AACnE;AAEA,eAAemC,sBAAsBJ,MAAoB,EAAEK,KAAiB,EAAEC,EAAU;IACtFN,OAAOd,EAAE,CAACN,IAAI,CAAC,CAAC,sBAAsB,EAAE0B,IAAI;AAC9C;AAEA,+FAA+F;AAC/F,OAAO,MAAMC,oBAA+C;IAC1DC,UAAUjC;IACVkC,eAAejC;IACfkC,kBAAkBtC;IAClByB;IACAI;IACAC;IACAE;IACA9B,aAAa,CAAC0B,QAAQrB,OAASL,YAAY0B,OAAOd,EAAE,EAAEP;AACxD,EAAE;AAEF,OAAO,eAAegC,WAAW7B,GAAmB;IAClD,OAAOhB,gBAAgBgB,KAAKyB;AAC9B"}

package/dist/esm/store/turso/connection.d.ts

@@ -1,4 +1,7 @@
 import type { Database } from '@tursodatabase/database';
 import type { Connection } from '../types.js';
 export declare function checkpointWal(db: Database): Promise<void>;
+export declare function cacheFilePath(conn: Connection): Promise<string | null>;
+export declare function fileSize(path: string): Promise<number | null>;
+export declare function reclaimSpace(path: string): Promise<void>;
 export declare function createConnection(db: Database): Connection;

package/dist/esm/store/turso/connection.js

@@ -1,4 +1,8 @@
+import { stat } from 'node:fs/promises';
+import { rewriteInsert } from '../batch.js';
+import { setMeta } from '../shared.js';
 import { BEGIN_WRITE, withTransaction } from '../transaction.js';
+import { CONNECT_OPTS, tursoApi } from './native.js';
 // @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%
 // faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.
 let TursoStatementWrapper = class TursoStatementWrapper {
@@ -33,6 +37,45 @@ export async function checkpointWal(db) {
         console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${err.message}`);
     }
 }
+// The main cache file's path, or null when unresolvable. The store holds a Database and
+// Connection, not a path (PLAN 3.52), so the path is read from the database itself.
+export async function cacheFilePath(conn) {
+    var _rows_find;
+    const rows = await (await conn.prepare('PRAGMA database_list')).all();
+    return ((_rows_find = rows.find((r)=>r.name === 'main')) === null || _rows_find === void 0 ? void 0 : _rows_find.file) || null;
+}
+export async function fileSize(path) {
+    try {
+        return (await stat(path)).size;
+    } catch  {
+        return null;
+    }
+}
+// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the
+// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).
+export async function reclaimSpace(path) {
+    try {
+        const turso = await tursoApi();
+        const db = await turso.connect(path, {
+            ...CONNECT_OPTS,
+            experimental: [
+                ...CONNECT_OPTS.experimental,
+                'vacuum'
+            ]
+        });
+        try {
+            await db.exec('VACUUM');
+            const size = await fileSize(path);
+            // Recorded here, not by the caller: this connection is the only one still open on the file.
+            if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));
+            await checkpointWal(db);
+        } finally{
+            await db.close();
+        }
+    } catch (err) {
+        console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${err.message}`);
+    }
+}
 export function createConnection(db) {
     const conn = {
         async exec (sql) {
@@ -41,14 +84,26 @@ export function createConnection(db) {
         async prepare (sql) {
             return new TursoStatementWrapper(await db.prepare(sql));
         },
-        // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as
-        // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.
+        // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):
+        // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{
-                const stmt = await db.prepare(sql);
+                const rewritten = rewriteInsert(sql, paramRows.length);
+                if (rewritten) {
+                    const stmt = await db.prepare(rewritten.sql);
+                    try {
+                        // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack
+                        // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).
+                        await stmt.run(paramRows.flat());
+                    } finally{
+                        await stmt.close();
+                    }
+                    return;
+                }
                 // Finalized here because nothing else will: open() hands back a connection the caller
                 // can hold across many batches, and the db.batch() this replaced finalized its own.
+                const stmt = await db.prepare(sql);
                 try {
                     for (const row of paramRows)await stmt.run(...row);
                 } finally{

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Prepares once and awaits run() per row: db.batch() re-prepares each statement, which cost as\n    // much as preparing per row. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = await db.prepare(sql);\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","createConnection","conn","sql","prepare","runBatch","paramRows","length","row","close"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,OAAO,SAASC,iBAAiBN,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKO,GAAW;YACpB,MAAMR,GAAGC,IAAI,CAACO;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIpB,sBAAsB,MAAMY,GAAGS,OAAO,CAACD;QACpD;QACA,+FAA+F;QAC/F,wKAAwK;QACxK,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMzB,gBACJoB,MACA;gBACE,MAAMhB,OAAO,MAAMS,GAAGS,OAAO,CAACD;gBAC9B,sFAAsF;gBACtF,oFAAoF;gBACpF,IAAI;oBACF,KAAK,MAAMK,OAAOF,UAAW,MAAMpB,KAAKF,GAAG,IAAIwB;gBACjD,SAAU;oBACR,MAAMtB,KAAKuB,KAAK;gBAClB;YACF,GACA5B;QAEJ;IACF;IACA,OAAOqB;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import { stat } from 'node:fs/promises';\nimport type { Database } from '@tursodatabase/database';\nimport { rewriteInsert } from '../batch.ts';\nimport { setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { CONNECT_OPTS, tursoApi } from './native.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// @tursodatabase/database/compat offers a synchronous escape hatch, but it measured only 4-12%\n// faster than this promise client, so this wraps the async client in the same Connection/Statement shape 1:1.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\n// This client leaves the WAL behind for the next opener, where node:sqlite checkpoints on the last\n// close, so a tree reconciled over and over grows one without bound. Best-effort: close must not throw.\nexport async function checkpointWal(db: Database): Promise<void> {\n  try {\n    await db.exec('PRAGMA wal_checkpoint(TRUNCATE)');\n  } catch (err) {\n    console.error(`sense: turso WAL checkpoint failed, the -wal file will keep growing until one succeeds: ${(err as Error).message}`);\n  }\n}\n\n// The main cache file's path, or null when unresolvable. The store holds a Database and\n// Connection, not a path (PLAN 3.52), so the path is read from the database itself.\nexport async function cacheFilePath(conn: Connection): Promise<string | null> {\n  const rows = (await (await conn.prepare('PRAGMA database_list')).all()) as Array<{ name: string; file: string }>;\n  return rows.find((r) => r.name === 'main')?.file || null;\n}\n\nexport async function fileSize(path: string): Promise<number | null> {\n  try {\n    return (await stat(path)).size;\n  } catch {\n    return null;\n  }\n}\n\n// Reclaims turso#8170's FTS space amplification (PLAN 3.41), on a throwaway connection after the\n// store's own has closed: `vacuum` doubles incremental reconcile cost merely by being enabled (3.54).\nexport async function reclaimSpace(path: string): Promise<void> {\n  try {\n    const turso = await tursoApi();\n    const db = await turso.connect(path, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental, 'vacuum'] });\n    try {\n      await db.exec('VACUUM');\n      const size = await fileSize(path);\n      // Recorded here, not by the caller: this connection is the only one still open on the file.\n      if (size !== null) await setMeta(createConnection(db), 'compact_size', String(size));\n      await checkpointWal(db);\n    } finally {\n      await db.close();\n    }\n  } catch (err) {\n    console.error(`sense: turso VACUUM failed, the cache will keep the space until one succeeds: ${(err as Error).message}`);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // Folds a plain INSERT into one multi-row VALUES statement (shared rewriteInsert, ../batch.ts):\n    // no bind-variable ceiling to chunk against, measured empirically. UPDATE/DELETE keep the per-row loop.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const rewritten = rewriteInsert(sql, paramRows.length);\n          if (rewritten) {\n            const stmt = await db.prepare(rewritten.sql);\n            try {\n              // One flat array, not spread: spreading tens of thousands of args hits a JS call-stack\n              // limit well before turso enforces any variable-count ceiling (measured; see PLAN.md).\n              await stmt.run(paramRows.flat());\n            } finally {\n              await stmt.close();\n            }\n            return;\n          }\n          // Finalized here because nothing else will: open() hands back a connection the caller\n          // can hold across many batches, and the db.batch() this replaced finalized its own.\n          const stmt = await db.prepare(sql);\n          try {\n            for (const row of paramRows) await stmt.run(...row);\n          } finally {\n            await stmt.close();\n          }\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["stat","rewriteInsert","setMeta","BEGIN_WRITE","withTransaction","CONNECT_OPTS","tursoApi","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","checkpointWal","db","exec","err","console","error","message","cacheFilePath","conn","rows","prepare","find","r","name","file","fileSize","path","size","reclaimSpace","turso","connect","experimental","createConnection","String","close","sql","runBatch","paramRows","length","rewritten","flat","row"],"mappings":"AAAA,SAASA,IAAI,QAAQ,mBAAmB;AAExC,SAASC,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,QAAQ,eAAe;AACvC,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,YAAY,EAAEC,QAAQ,QAAQ,cAAc;AAMrD,+FAA+F;AAC/F,8GAA8G;AAC9G,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,mGAAmG;AACnG,wGAAwG;AACxG,OAAO,eAAeQ,cAAcC,EAAY;IAC9C,IAAI;QACF,MAAMA,GAAGC,IAAI,CAAC;IAChB,EAAE,OAAOC,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,wFAAwF,EAAE,AAACF,IAAcG,OAAO,EAAE;IACnI;AACF;AAEA,wFAAwF;AACxF,oFAAoF;AACpF,OAAO,eAAeC,cAAcC,IAAgB;QAE3CC;IADP,MAAMA,OAAQ,MAAM,AAAC,CAAA,MAAMD,KAAKE,OAAO,CAAC,uBAAsB,EAAGhB,GAAG;IACpE,OAAOe,EAAAA,aAAAA,KAAKE,IAAI,CAAC,CAACC,IAAMA,EAAEC,IAAI,KAAK,qBAA5BJ,iCAAAA,WAAqCK,IAAI,KAAI;AACtD;AAEA,OAAO,eAAeC,SAASC,IAAY;IACzC,IAAI;QACF,OAAO,AAAC,CAAA,MAAMlC,KAAKkC,KAAI,EAAGC,IAAI;IAChC,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,iGAAiG;AACjG,sGAAsG;AACtG,OAAO,eAAeC,aAAaF,IAAY;IAC7C,IAAI;QACF,MAAMG,QAAQ,MAAM/B;QACpB,MAAMa,KAAK,MAAMkB,MAAMC,OAAO,CAACJ,MAAM;YAAE,GAAG7B,YAAY;YAAEkC,cAAc;mBAAIlC,aAAakC,YAAY;gBAAE;aAAS;QAAC;QAC/G,IAAI;YACF,MAAMpB,GAAGC,IAAI,CAAC;YACd,MAAMe,OAAO,MAAMF,SAASC;YAC5B,4FAA4F;YAC5F,IAAIC,SAAS,MAAM,MAAMjC,QAAQsC,iBAAiBrB,KAAK,gBAAgBsB,OAAON;YAC9E,MAAMjB,cAAcC;QACtB,SAAU;YACR,MAAMA,GAAGuB,KAAK;QAChB;IACF,EAAE,OAAOrB,KAAK;QACZC,QAAQC,KAAK,CAAC,CAAC,8EAA8E,EAAE,AAACF,IAAcG,OAAO,EAAE;IACzH;AACF;AAEA,OAAO,SAASgB,iBAAiBrB,EAAY;IAC3C,MAAMO,OAAmB;QACvB,MAAMN,MAAKuB,GAAW;YACpB,MAAMxB,GAAGC,IAAI,CAACuB;QAChB;QACA,MAAMf,SAAQe,GAAW;YACvB,OAAO,IAAIpC,sBAAsB,MAAMY,GAAGS,OAAO,CAACe;QACpD;QACA,gGAAgG;QAChG,wGAAwG;QACxG,MAAMC,UAASD,GAAW,EAAEE,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAM1C,gBACJsB,MACA;gBACE,MAAMqB,YAAY9C,cAAc0C,KAAKE,UAAUC,MAAM;gBACrD,IAAIC,WAAW;oBACb,MAAMrC,OAAO,MAAMS,GAAGS,OAAO,CAACmB,UAAUJ,GAAG;oBAC3C,IAAI;wBACF,uFAAuF;wBACvF,uFAAuF;wBACvF,MAAMjC,KAAKF,GAAG,CAACqC,UAAUG,IAAI;oBAC/B,SAAU;wBACR,MAAMtC,KAAKgC,KAAK;oBAClB;oBACA;gBACF;gBACA,sFAAsF;gBACtF,oFAAoF;gBACpF,MAAMhC,OAAO,MAAMS,GAAGS,OAAO,CAACe;gBAC9B,IAAI;oBACF,KAAK,MAAMM,OAAOJ,UAAW,MAAMnC,KAAKF,GAAG,IAAIyC;gBACjD,SAAU;oBACR,MAAMvC,KAAKgC,KAAK;gBAClB;YACF,GACAvC;QAEJ;IACF;IACA,OAAOuB;AACT"}

package/dist/esm/store/turso/native.d.ts

@@ -1,2 +1,6 @@
 export declare const TURSO_PACKAGE = "@tursodatabase/database";
+export declare const CONNECT_OPTS: {
+    readonly timeout: 30000;
+    readonly experimental: readonly ["index_method"];
+};
 export declare function tursoApi(): Promise<typeof import('@tursodatabase/database')>;

package/dist/esm/store/turso/native.js

@@ -5,6 +5,14 @@ const TURSO = {
     pkg: TURSO_PACKAGE,
     sizeHint: '~16MB'
 };
+// Every connect() in this store goes through these: the same 30s floor sqlite opens with, and the
+// FTS index method ensureSchema() needs. reclaimSpace adds 'vacuum'; nothing else may (PLAN 3.54).
+export const CONNECT_OPTS = {
+    timeout: 30000,
+    experimental: [
+        'index_method'
+    ]
+};
 let tursoApiPromise;
 // One accessor every consumer must go through, not each its own bare-specifier import: Node
 // keeps returning a pre-install "not found" miss to any new lookup of the same path afterward.

package/dist/esm/store/turso/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,IAAIC;AAEJ,4FAA4F;AAC5F,+FAA+F;AAC/F,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBP,cAAwDG,OAAOF;IACvG,OAAOM;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const TURSO_PACKAGE = '@tursodatabase/database';\n\nconst TURSO: NativeDescriptor = { store: 'turso', pkg: TURSO_PACKAGE, sizeHint: '~16MB' };\n\n// Every connect() in this store goes through these: the same 30s floor sqlite opens with, and the\n// FTS index method ensureSchema() needs. reclaimSpace adds 'vacuum'; nothing else may (PLAN 3.54).\nexport const CONNECT_OPTS = { timeout: 30_000, experimental: ['index_method'] } as const;\n\nlet tursoApiPromise: Promise<typeof import('@tursodatabase/database')> | undefined;\n\n// One accessor every consumer must go through, not each its own bare-specifier import: Node\n// keeps returning a pre-install \"not found\" miss to any new lookup of the same path afterward.\nexport function tursoApi(): Promise<typeof import('@tursodatabase/database')> {\n  if (!tursoApiPromise) tursoApiPromise = loadOrInstall<typeof import('@tursodatabase/database')>(TURSO, packageNodeModules());\n  return tursoApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","TURSO_PACKAGE","TURSO","store","pkg","sizeHint","CONNECT_OPTS","timeout","experimental","tursoApiPromise","tursoApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,gBAAgB,0BAA0B;AAEvD,MAAMC,QAA0B;IAAEC,OAAO;IAASC,KAAKH;IAAeI,UAAU;AAAQ;AAExF,kGAAkG;AAClG,mGAAmG;AACnG,OAAO,MAAMC,eAAe;IAAEC,SAAS;IAAQC,cAAc;QAAC;KAAe;AAAC,EAAW;AAEzF,IAAIC;AAEJ,4FAA4F;AAC5F,+FAA+F;AAC/F,OAAO,SAASC;IACd,IAAI,CAACD,iBAAiBA,kBAAkBV,cAAwDG,OAAOF;IACvG,OAAOS;AACT"}

package/dist/esm/store/turso/open.d.ts

@@ -3,7 +3,7 @@ import type { ResolvedConfig } from '../../config/index.js';
 import type { OpenResult } from '../open.js';
 import type { OpenDialect } from '../types.js';
 export declare const DB_FILENAME = "cache.turso.db";
-export declare const SCHEMA_VERSION = "4";
+export declare const SCHEMA_VERSION = "5";
 export type { OpenResult };
 export declare const tursoOpenDialect: OpenDialect<Database>;
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -5,13 +5,13 @@ import { activeFeatures, FEATURES } from '../../features/index.js';
 import { openWithDialect } from '../open.js';
 import { getMeta, setMeta } from '../shared.js';
 import { checkpointWal, createConnection } from './connection.js';
-import { TURSO_PACKAGE, tursoApi } from './native.js';
+import { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.js';
 import { CONTENT_FTS_DDL, tursoDialect } from './reconcile.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.turso.db';
 // Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves
 // separately. Covers the FTS indexes, the "_ngram" sidecar columns, and embeddings.vector's width.
-export const SCHEMA_VERSION = '4';
+export const SCHEMA_VERSION = '5';
 // This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no
 // extra native state to thread, unlike duckdb's separate instance/connection pair.
 async function ensureSchema(_handle, conn, cfg) {
@@ -51,12 +51,12 @@ async function connect(dbPath, _cfg) {
     }
     let db;
     try {
-        // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;
-        // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).
+        // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA
+        // below. Options in native.ts, shared with reclaimSpace's own connection.
         db = await turso.connect(dbPath, {
-            timeout: 30000,
+            ...CONNECT_OPTS,
             experimental: [
-                'index_method'
+                ...CONNECT_OPTS.experimental
             ]
         });
     } catch (err) {

package/dist/esm/store/turso/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '4';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // Floored at the same 30s sqlite opens with. `timeout` is connect-time only in this client;\n    // the derived value is set via runtime PRAGMA below. `index_method` is required for ensureSchema()'s FTS indexes (T1).\n    db = await turso.connect(dbPath, { timeout: 30_000, experimental: ['index_method'] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","timeout","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACtD,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWpB,eAAeiB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAEpB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMsB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMb,QAAQa,MAAM,sBAAuB,MAAM,MAAMZ,QAAQY,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMV,QAAQa,MAAM,gBAAiB,MAAM,MAAMZ,QAAQY,MAAM,YAAYnB,iBAAiBoB,KAAKhB;AACxG;AAEA,eAAesB,MAAMC,MAAgB;IACnC,MAAMnB,cAAcmB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAehC,YAAY,MAAMgC;QACrC,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,wBAAwB,EAAEQ,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,4FAA4F;QAC5F,uHAAuH;QACvHA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAEM,SAAS;YAAQC,cAAc;gBAAC;aAAe;QAAC;IACrF,EAAE,OAAOJ,KAAK;QACZ,MAAM,IAAIhC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE6B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMV,iBAAiB2B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAMG,mBAA0C;IACrDC,UAAUzB;IACV0B,eAAezB;IACf0B,kBAAkB7B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGiB,UAAU,CAACT,MAAQ,iEAAiEU,IAAI,CAACV,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAe0B,UAAUzB,GAAmB;IACjD,OAAOf,gBAAgBe,KAAKmB;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/open.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { checkpointWal, createConnection } from './connection.ts';\nimport { CONNECT_OPTS, TURSO_PACKAGE, tursoApi } from './native.ts';\nimport { CONTENT_FTS_DDL, tursoDialect } from './reconcile.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.turso.db';\n// Independent of sqlite's and duckdb's SCHEMA_VERSION: each store's cache shape evolves\n// separately. Covers the FTS indexes, the \"_ngram\" sidecar columns, and embeddings.vector's width.\nexport const SCHEMA_VERSION = '5';\n\nexport type { OpenResult };\n\n// This store's Handle (types.ts's OpenDialect<Handle>) is the connected Database itself: no\n// extra native state to thread, unlike duckdb's separate instance/connection pair.\nasync function ensureSchema(_handle: Database, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);\n  for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native F32_BLOB(STORE_DIMS) instead of the embed feature's engine-neutral BLOB DDL. `scale`\n    // is kept unused, so the shared reconcile-time INSERT/DELETE names a column both stores have.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector F32_BLOB(${STORE_DIMS}), PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function close(handle: Database): Promise<void> {\n  await checkpointWal(handle);\n  await handle.close();\n}\n\nasync function setDerivedBusyTimeout(_handle: Database, conn: Connection, ms: number): Promise<void> {\n  await conn.exec(`PRAGMA busy_timeout = ${ms}`);\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: Database; conn: Connection }> {\n  // Dynamic, not a top-level import: a sqlite or duckdb tree must never attempt to resolve this\n  // optional dependency until a turso tree is actually opened. Installed on first use if missing.\n  let turso: Awaited<ReturnType<typeof tursoApi>>;\n  try {\n    turso = await tursoApi();\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" needs the ${TURSO_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let db: Database;\n  try {\n    // `timeout` is connect-time only in this client; the derived value is set via runtime PRAGMA\n    // below. Options in native.ts, shared with reclaimSpace's own connection.\n    db = await turso.connect(dbPath, { ...CONNECT_OPTS, experimental: [...CONNECT_OPTS.experimental] });\n  } catch (err) {\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"turso\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  return { handle: db, conn: createConnection(db) };\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const tursoOpenDialect: OpenDialect<Database> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: tursoDialect,\n  connect,\n  close,\n  // \"Locking error: Failed locking file ...\", worded per platform: posix \"File is locked by another\n  // process\", Windows \"another process has locked a portion of the file (os error 33)\". Distinct from\n  // the write-time \"database is locked\" its connect-time `timeout` covers; that one never reaches here.\n  isLocked: (err) => /File is locked by another process|locked a portion of the file/.test(err.message),\n  ensureSchema,\n  setDerivedBusyTimeout,\n  createStore: (handle, conn) => createStore(handle, conn),\n};\n\nexport async function openTurso(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, tursoOpenDialect);\n}\n"],"names":["featureSignature","STORE_DIMS","SenseError","activeFeatures","FEATURES","openWithDialect","getMeta","setMeta","checkpointWal","createConnection","CONNECT_OPTS","TURSO_PACKAGE","tursoApi","CONTENT_FTS_DDL","tursoDialect","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","_handle","conn","cfg","exec","ddl","feature","name","schema","close","handle","setDerivedBusyTimeout","ms","connect","dbPath","_cfg","turso","err","message","db","experimental","tursoOpenDialect","filename","schemaVersion","reconcileDialect","isLocked","test","openTurso"],"mappings":"AAEA,SAASA,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AAEnE,SAASC,eAAe,QAAQ,aAAa;AAC7C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,aAAa,EAAEC,gBAAgB,QAAQ,kBAAkB;AAClE,SAASC,YAAY,EAAEC,aAAa,EAAEC,QAAQ,QAAQ,cAAc;AACpE,SAASC,eAAe,EAAEC,YAAY,QAAQ,iBAAiB;AAC/D,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,iBAAiB;AAC5C,wFAAwF;AACxF,mGAAmG;AACnG,OAAO,MAAMC,iBAAiB,IAAI;AAIlC,4FAA4F;AAC5F,mFAAmF;AACnF,eAAeC,aAAaC,OAAiB,EAAEC,IAAgB,EAAEC,GAAW;IAC1E,MAAMD,KAAKE,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,MAAMF,KAAKE,IAAI,CAAC,CAAC,wJAAwJ,CAAC;IAC1K,KAAK,MAAMC,OAAOV,gBAAiB,MAAMO,KAAKE,IAAI,CAACC;IACnD,MAAMH,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAME,WAAWrB,eAAekB,KAAM;QACzC,8FAA8F;QAC9F,8FAA8F;QAC9F,IAAIG,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAML,KAAKE,IAAI,CAAC,CAAC,qIAAqI,EAAErB,WAAW,+BAA+B,CAAC;YACnM;QACF;QACA,MAAMuB,QAAQE,MAAM,CAACN;IACvB;IACA,IAAI,AAAC,MAAMd,QAAQc,MAAM,sBAAuB,MAAM,MAAMb,QAAQa,MAAM,kBAAkBH;IAC5F,IAAI,AAAC,MAAMX,QAAQc,MAAM,gBAAiB,MAAM,MAAMb,QAAQa,MAAM,YAAYpB,iBAAiBqB,KAAKjB;AACxG;AAEA,eAAeuB,MAAMC,MAAgB;IACnC,MAAMpB,cAAcoB;IACpB,MAAMA,OAAOD,KAAK;AACpB;AAEA,eAAeE,sBAAsBV,OAAiB,EAAEC,IAAgB,EAAEU,EAAU;IAClF,MAAMV,KAAKE,IAAI,CAAC,CAAC,sBAAsB,EAAEQ,IAAI;AAC/C;AAEA,eAAeC,QAAQC,MAAc,EAAEC,IAAoB;IACzD,8FAA8F;IAC9F,gGAAgG;IAChG,IAAIC;IACJ,IAAI;QACFA,QAAQ,MAAMtB;IAChB,EAAE,OAAOuB,KAAK;QACZ,IAAIA,eAAejC,YAAY,MAAMiC;QACrC,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,wBAAwB,EAAES,cAAc,UAAU,EAAE,AAACwB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACjI;IAEA,IAAIC;IACJ,IAAI;QACF,6FAA6F;QAC7F,0EAA0E;QAC1EA,KAAK,MAAMH,MAAMH,OAAO,CAACC,QAAQ;YAAE,GAAGtB,YAAY;YAAE4B,cAAc;mBAAI5B,aAAa4B,YAAY;aAAC;QAAC;IACnG,EAAE,OAAOH,KAAK;QACZ,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,6BAA6B,EAAE8B,OAAO,EAAE,EAAE,AAACG,IAAcC,OAAO,EAAE;IACtH;IACA,OAAO;QAAER,QAAQS;QAAIjB,MAAMX,iBAAiB4B;IAAI;AAClD;AAEA,+FAA+F;AAC/F,OAAO,MAAME,mBAA0C;IACrDC,UAAUxB;IACVyB,eAAexB;IACfyB,kBAAkB5B;IAClBiB;IACAJ;IACA,kGAAkG;IAClG,oGAAoG;IACpG,sGAAsG;IACtGgB,UAAU,CAACR,MAAQ,iEAAiES,IAAI,CAACT,IAAIC,OAAO;IACpGlB;IACAW;IACAd,aAAa,CAACa,QAAQR,OAASL,YAAYa,QAAQR;AACrD,EAAE;AAEF,OAAO,eAAeyB,UAAUxB,GAAmB;IACjD,OAAOhB,gBAAgBgB,KAAKkB;AAC9B"}

package/dist/esm/store/turso/store.js

@@ -1,8 +1,8 @@
 import { STORE_DIMS } from '../../embed/types.js';
-import { getColumns } from '../shared.js';
+import { getColumns, getMeta, setMeta } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
-import { checkpointWal } from './connection.js';
+import { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { queryLexical } from './lexical.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
@@ -13,6 +13,9 @@ export const CAPABILITIES = new Set([
     'phrases',
     'vectors'
 ]);
+// close() reclaims once the cache file outgrows its compact size by this factor: the disk
+// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.
+const BLOAT_FACTOR = 1.5;
 // Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth
 // (see transaction.ts) is tracked against the same object everywhere.
 export function createStore(db, conn) {
@@ -73,8 +76,32 @@ export function createStore(db, conn) {
             }
         },
         async close () {
+            // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.
             await checkpointWal(db);
+            let reclaimPath = null;
+            try {
+                var _ref;
+                const path = await cacheFilePath(conn);
+                const base = Number((_ref = await getMeta(conn, 'compact_size')) !== null && _ref !== void 0 ? _ref : '0');
+                const size = path === null ? null : await fileSize(path);
+                if (size !== null) {
+                    // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a
+                    // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.
+                    if (base === 0) {
+                        await setMeta(conn, 'compact_size', String(size));
+                        // The meta write landed past the checkpoint; leave no WAL for the next opener.
+                        await checkpointWal(db);
+                    } else if (size > BLOAT_FACTOR * base) {
+                        reclaimPath = path;
+                    }
+                }
+            } catch (err) {
+                console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${err.message}`);
+            }
             await db.close();
+            // After this store's connection is closed, never before: turso hands the file to one
+            // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).
+            if (reclaimPath !== null) await reclaimSpace(reclaimPath);
         }
     };
 }

package/dist/esm/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { checkpointWal } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await checkpointWal(db);\n      await db.close();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","checkpointWal","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,aAAa,QAAQ,kBAAkB;AAChD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,2BAA2B;AAC3B,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcN;QACd,MAAMO,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOtB,gBAAgBa,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMzB,WAAWc;iBAAO;YACtC;YACAT,YAAY,CAACoB,SAASC,aAAerB,WAAWS,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAASxB,aAAaQ,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAM7B,YAAYW;YAC3BmB,cAAc,CAACC,OAASzB,iBAAiBK,MAAMf,YAAYmC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYhC,eAAeO,MAAMsB,IAAIrC,YAAYuC,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAAStB,YAAYM,MAAMf,YAAY0C,MAAMX;YAC7DY,WAAW,CAACD,OAASvC,aAAaY,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,MAAMhD,cAAcS;YACpB,MAAMA,GAAGuC,KAAK;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns, getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { cacheFilePath, checkpointWal, fileSize, reclaimSpace } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// close() reclaims once the cache file outgrows its compact size by this factor: the disk\n// overhead a user should accept (PLAN 3.52), in postgres autovacuum scale-factor shape.\nconst BLOAT_FACTOR = 1.5;\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      // Checkpoint before measuring: the last reconcile's pages sit in the WAL, not the file yet.\n      await checkpointWal(db);\n      let reclaimPath: string | null = null;\n      try {\n        const path = await cacheFilePath(conn);\n        const base = Number((await getMeta(conn, 'compact_size')) ?? '0');\n        const size = path === null ? null : await fileSize(path);\n        if (size !== null) {\n          // No baseline yet (cold build or an upgraded cache): record it, never reclaim here, so a\n          // cold build pays no VACUUM (PLAN 3.49 Track A) and an upgraded cache heals on a crossing.\n          if (base === 0) {\n            await setMeta(conn, 'compact_size', String(size));\n            // The meta write landed past the checkpoint; leave no WAL for the next opener.\n            await checkpointWal(db);\n          } else if (size > BLOAT_FACTOR * base) {\n            reclaimPath = path;\n          }\n        }\n      } catch (err) {\n        console.error(`sense: turso cache bloat check failed, the cache will keep its space until one succeeds: ${(err as Error).message}`);\n      }\n      await db.close();\n      // After this store's connection is closed, never before: turso hands the file to one\n      // connection at a time, and reclaimSpace opens its own to carry the `vacuum` flag (PLAN 3.54).\n      if (reclaimPath !== null) await reclaimSpace(reclaimPath);\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","getMeta","setMeta","withTransaction","hasVectorRow","pendingRows","cacheFilePath","checkpointWal","fileSize","reclaimSpace","fieldStats","queryLexical","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","BLOAT_FACTOR","createStore","db","conn","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","docs","columns","scopeWhere","lexical","query","terms","opts","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","hasVector","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close","reclaimPath","base","Number","size","String","err","console","error","message"],"mappings":"AACA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,EAAEC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAC5D,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,aAAa,EAAEC,aAAa,EAAEC,QAAQ,EAAEC,YAAY,QAAQ,kBAAkB;AACvF,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,YAAY,QAAQ,eAAe;AAC5C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,2BAA2B;AAC3B,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,0FAA0F;AAC1F,wFAAwF;AACxF,MAAMC,eAAe;AAErB,mGAAmG;AACnG,sEAAsE;AACtE,OAAO,SAASC,YAAYC,EAAY,EAAEC,IAAgB;IACxD,OAAO;QACLC,MAAM;QACNC,cAAcP;QACd,MAAMQ,MAAKC,GAAW;YACpB,MAAMJ,KAAKG,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOJ,KAAKK,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMP,KAAKM,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO1B,gBAAgBiB,MAAMS;QAC/B;QACAC,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM/B,WAAWoB;iBAAO;YACtC;YACAV,YAAY,CAACqB,SAASC,aAAetB,WAAWU,MAAMW,SAASC;QACjE;QACAC,SAAS;YACPC,OAAO,CAACC,OAAOC,OAASzB,aAAaS,MAAMe,OAAOC;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIC,SAAS;YACPC,SAAS,IAAMjC,YAAYe;YAC3BmB,cAAc,CAACC,OAAS1B,iBAAiBM,MAAMrB,YAAYyC;YAC3DC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYjC,eAAeQ,MAAMsB,IAAI3C,YAAY6C,OAAOC;YAC5FC,SAAS,CAACC,MAAMX,OAASvB,YAAYO,MAAMrB,YAAYgD,MAAMX;YAC7DY,WAAW,CAACD,OAAS3C,aAAagB,MAAM2B;QAC1C;QACA,MAAME;YACJ,qFAAqF;YACrF,sKAAsK;YACtK,MAAMC,MAAO,MAAM,AAAC,CAAA,MAAM/B,GAAGM,OAAO,CAAC,sBAAqB,EAAG0B,GAAG;YAChE,OAAO;gBAAEC,cAAc,GAAGF,IAAIE,YAAY,CAAC,kFAAkF,CAAC;YAAC;QACjI;QACAC,KAAK;YACH,MAAM5B,SAAQD,GAAW;gBACvB,MAAM8B,OAAO,MAAMnC,GAAGM,OAAO,CAACD;gBAC9B8B,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gBACzF,OAAO;oBACLxB,SAAS,IAAMuB,KAAKvB,OAAO;oBAC3B,8DAA8D;oBAC9DyB,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,OAAOH,KAAKE,OAAO,IAAIC;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,4FAA4F;YAC5F,MAAMnD,cAAcY;YACpB,IAAIwC,cAA6B;YACjC,IAAI;oBAEmB;gBADrB,MAAMZ,OAAO,MAAMzC,cAAcc;gBACjC,MAAMwC,OAAOC,QAAQ,OAAA,MAAM5D,QAAQmB,MAAM,6BAApB,kBAAA,OAAwC;gBAC7D,MAAM0C,OAAOf,SAAS,OAAO,OAAO,MAAMvC,SAASuC;gBACnD,IAAIe,SAAS,MAAM;oBACjB,yFAAyF;oBACzF,2FAA2F;oBAC3F,IAAIF,SAAS,GAAG;wBACd,MAAM1D,QAAQkB,MAAM,gBAAgB2C,OAAOD;wBAC3C,+EAA+E;wBAC/E,MAAMvD,cAAcY;oBACtB,OAAO,IAAI2C,OAAO7C,eAAe2C,MAAM;wBACrCD,cAAcZ;oBAChB;gBACF;YACF,EAAE,OAAOiB,KAAK;gBACZC,QAAQC,KAAK,CAAC,CAAC,yFAAyF,EAAE,AAACF,IAAcG,OAAO,EAAE;YACpI;YACA,MAAMhD,GAAGuC,KAAK;YACd,qFAAqF;YACrF,+FAA+F;YAC/F,IAAIC,gBAAgB,MAAM,MAAMlD,aAAakD;QAC/C;IACF;AACF"}

package/dist/esm/text/strip.js

@@ -3,7 +3,7 @@ import { normalizeText } from '../scan/frontmatter.js';
 // Flat prose from blocks a caller already parsed, via the same extractor the chunker uses
 // (src/chunk) -- words and vectors read one definition of "the prose of this note".
 export function textFromBlocks(blocks) {
-    return normalizeText(extractTexts(blocks.map((block)=>block.node)));
+    return normalizeText(extractTexts(blocks));
 }
 // Thin parse + delegate, for callers with no blocks of their own already.
 export function stripText(value) {

package/dist/esm/text/strip.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks.map((block) => block.node)));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","map","block","node","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI,OAAOC,GAAG,CAAC,CAACC,QAAUA,MAAMC,IAAI;AACpE;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAON,eAAeF,MAAMQ;AAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/text/strip.ts"],"sourcesContent":["import type { Block } from '../chunk/index.ts';\nimport { extractTexts, parse } from '../chunk/index.ts';\nimport { normalizeText } from '../scan/frontmatter.ts';\n\n// Flat prose from blocks a caller already parsed, via the same extractor the chunker uses\n// (src/chunk) -- words and vectors read one definition of \"the prose of this note\".\nexport function textFromBlocks(blocks: Block[]): string {\n  return normalizeText(extractTexts(blocks));\n}\n\n// Thin parse + delegate, for callers with no blocks of their own already.\nexport function stripText(value: string): string {\n  return textFromBlocks(parse(value));\n}\n"],"names":["extractTexts","parse","normalizeText","textFromBlocks","blocks","stripText","value"],"mappings":"AACA,SAASA,YAAY,EAAEC,KAAK,QAAQ,oBAAoB;AACxD,SAASC,aAAa,QAAQ,yBAAyB;AAEvD,0FAA0F;AAC1F,oFAAoF;AACpF,OAAO,SAASC,eAAeC,MAAe;IAC5C,OAAOF,cAAcF,aAAaI;AACpC;AAEA,0EAA0E;AAC1E,OAAO,SAASC,UAAUC,KAAa;IACrC,OAAOH,eAAeF,MAAMK;AAC9B"}

package/dist/esm/workers/parse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the mdast tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","start","process","hrtime","bigint","doc","warnings","parseMs","Number","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAiBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGlB,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,MAAMgB,UAAUC,OAAON,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEQ,IAAI;YAAMJ;YAAKC;YAAUC;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOrB,eAAeoB;QAAK;IACjD;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/workers/parse.ts"],"sourcesContent":["import Tinypool from 'tinypool';\nimport type { Config, FeatureName } from '../config/index.ts';\nimport { FEATURES } from '../features/index.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { FileStat } from '../scan/list.ts';\nimport { featuresForFile } from '../scan/reparse.ts';\nimport type { WorkerErrorPayload } from '../scan/worker-error.ts';\nimport { serializeError } from '../scan/worker-error.ts';\n\n// Constant for the whole dispatch, so it crosses once per worker instead of once per task. A\n// Feature carries closures and cannot cross the thread boundary; its name can, and the registry here resolves it back.\nexport interface ParseWorkerData {\n  cfg: Config;\n  featureNames: FeatureName[];\n}\n\n// tinypool's task, in and out. The task itself is one FileStat. Result carries only what\n// parseFile already returns -- extracted text and per-feature values, never the token tree.\nexport type ParseTask = FileStat;\n\n// parseMs is the worker's own hrtime for parseFile, excluding dispatch and the return clone --\n// what separates it from the pool's dispatch-to-drain `parse` stage.\nexport type ParseTaskResult = { ok: true; doc: ParsedDoc; warnings: string[]; parseMs: number } | { ok: false; error: WorkerErrorPayload };\n\n// Read once per worker, not per task.\nconst { cfg, featureNames } = Tinypool.workerData as ParseWorkerData;\n// Filtering the registry (rather than mapping the names) keeps registry order, which is the\n// order `extracted` keys land in on the serial path.\nconst selected = FEATURES.filter((feature) => featureNames.includes(feature.name));\n\nexport default function parseTask(file: ParseTask): ParseTaskResult {\n  try {\n    const start = process.hrtime.bigint();\n    const { doc, warnings } = parseFile(file, featuresForFile(selected, cfg, file), cfg);\n    const parseMs = Number(process.hrtime.bigint() - start) / 1e6;\n    return { ok: true, doc, warnings, parseMs };\n  } catch (err) {\n    return { ok: false, error: serializeError(err) };\n  }\n}\n"],"names":["Tinypool","FEATURES","parseFile","featuresForFile","serializeError","cfg","featureNames","workerData","selected","filter","feature","includes","name","parseTask","file","start","process","hrtime","bigint","doc","warnings","parseMs","Number","ok","err","error"],"mappings":"AAAA,OAAOA,cAAc,WAAW;AAEhC,SAASC,QAAQ,QAAQ,uBAAuB;AAEhD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,eAAe,QAAQ,qBAAqB;AAErD,SAASC,cAAc,QAAQ,0BAA0B;AAiBzD,sCAAsC;AACtC,MAAM,EAAEC,GAAG,EAAEC,YAAY,EAAE,GAAGN,SAASO,UAAU;AACjD,4FAA4F;AAC5F,qDAAqD;AACrD,MAAMC,WAAWP,SAASQ,MAAM,CAAC,CAACC,UAAYJ,aAAaK,QAAQ,CAACD,QAAQE,IAAI;AAEhF,eAAe,SAASC,UAAUC,IAAe;IAC/C,IAAI;QACF,MAAMC,QAAQC,QAAQC,MAAM,CAACC,MAAM;QACnC,MAAM,EAAEC,GAAG,EAAEC,QAAQ,EAAE,GAAGlB,UAAUY,MAAMX,gBAAgBK,UAAUH,KAAKS,OAAOT;QAChF,MAAMgB,UAAUC,OAAON,QAAQC,MAAM,CAACC,MAAM,KAAKH,SAAS;QAC1D,OAAO;YAAEQ,IAAI;YAAMJ;YAAKC;YAAUC;QAAQ;IAC5C,EAAE,OAAOG,KAAK;QACZ,OAAO;YAAED,IAAI;YAAOE,OAAOrB,eAAeoB;QAAK;IACjD;AACF"}

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "sensemaking",
-  "version": "0.22.2",
+  "version": "0.23.1",
   "description": "Query and search your markdown notes with context-aware progressive disclosure: SQL over frontmatter, links, and text, plus semantic search and link-graph ranking. No server, no build step",
   "keywords": [
     "markdown",
@@ -84,24 +84,16 @@
     "@huggingface/tokenizers": "^0.1.3",
     "franc-min": "^6.2.0",
     "install-module-linked": "^2.0.0",
-    "mdast-util-from-markdown": "^2.0.3",
-    "mdast-util-gfm-autolink-literal": "^2.0.0",
-    "mdast-util-gfm-footnote": "^2.0.0",
-    "mdast-util-gfm-strikethrough": "^2.0.0",
-    "mdast-util-gfm-table": "^2.0.0",
-    "mdast-util-gfm-task-list-item": "^2.0.0",
-    "micromark-extension-gfm-autolink-literal": "^2.0.0",
-    "micromark-extension-gfm-footnote": "^2.0.0",
-    "micromark-extension-gfm-strikethrough": "^2.0.0",
-    "micromark-extension-gfm-table": "^2.0.0",
-    "micromark-extension-gfm-task-list-item": "^2.0.0",
+    "markdown-it": "^15.0.1",
+    "markdown-it-footnote": "^4.0.0",
+    "markdown-it-task-lists": "^2.1.1",
     "tinypool": "^2.1.2",
     "yaml": "^2.9.0"
   },
   "devDependencies": {
     "@duckdb/node-api": "*",
     "@tursodatabase/database": "*",
-    "@types/mdast": "^4.0.4",
+    "@types/markdown-it-footnote": "^3.0.4",
     "@types/mocha": "*",
     "@types/node": "*",
     "cr": "^0.1.0",

package/skills/sense-setup/SKILL.md

@@ -10,7 +10,7 @@ Querying an existing tree is the `sense` skill. This one covers making a tree: i
 ## Setup
 
 - `npm install -g sensemaking`, then `sense init` at the tree root writes `sense.config.json`: two presets (`default`, and `large` showing what a big tree tunes) and an `embed` block naming the model. The model fetches once per machine at the first vector search (progress on stderr); `sense download` prefetches it instead where that timing matters (CI, air-gapped setup). Config discovery walks up from cwd; `--config <path>` overrides.
-- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`, whose engine package the first command that opens such a tree installs on its own (`@duckdb/node-api`, a one-time native download of about 110 MB; `@tursodatabase/database`, much smaller). The same commands and tables run on all three. Two things do not port, and each one decides a tree. **FTS5 syntax:** under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect; a tree whose saved queries or search vocabulary depend on FTS5 operators stays on `sqlite`. **SQL functions:** `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register them at all, so a tree whose queries call them stays off `turso`. `sense watch` runs on all three; `duckdb` and `turso` lock their cache file per connection, so a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. Indexing speed is the other axis, and it does not follow from any of the above: sqlite builds a cold index fastest and turso slowest, by a wide margin on a large tree, and no setting closes that gap, since turso's engine costs more per write and more again to maintain each index. What turso buys instead is concurrent writers, non-blocking I/O and encryption, none of which a one-shot command uses, so reach for it for those rather than for speed. Per-store figures: BENCHMARKING.md in the sensemaking repo.
+- **Backing store.** The config's `store` key: `sqlite` (default, zero-dependency, Node's built-in SQLite), or the experimental `duckdb` and `turso`, whose engine package the first command that opens such a tree installs on its own (`@duckdb/node-api`, a one-time native download of about 110 MB; `@tursodatabase/database`, much smaller). The same commands and tables run on all three. Two things do not port, and each one decides a tree. **FTS5 syntax:** under `duckdb` and `turso`, `search` text and raw `MATCH` reject FTS5's prefix, boolean, `NEAR`, initial-token and column-filter operators with a named error, and sqlite's FTS5 SQL (`MATCH`, `snippet()`, `bm25()`) does not run, so saved queries written in that syntax are sqlite dialect; a tree whose saved queries or search vocabulary depend on FTS5 operators stays on `sqlite`. **SQL functions:** `has`/`basename`/`segment` are registered on `sqlite` and `duckdb` but not `turso`, whose client cannot register them at all, so a tree whose queries call them stays off `turso`. `sense watch` runs on all three; `duckdb` and `turso` lock their cache file per connection, so a concurrent command waits out the watcher's current cycle instead of failing. Each store keeps its own cache file (`.sense/cache.db`, `.sense/cache.duckdb`, `.sense/cache.turso.db`); switching stores is a rebuild, not a migration. Indexing speed is the other axis, and it does not follow from any of the above: sqlite builds a cold index fastest and turso slowest, by a wide margin on a large tree, and no setting closes that gap, since turso's engine costs more per write and more again to maintain each index. What turso buys instead is concurrent access, non-blocking I/O and encryption, so reach for it for those rather than for speed. Per-store figures: BENCHMARKING.md in the sensemaking repo.
 - Globs resolve relative to the config file, never the cwd.
 - `sense status` and `sense map` show each preset's coverage (files matched, embedded count), so what a config actually indexes is always visible in output. A config edit that changes coverage rebuilds the cache and names the preset that caused it on stderr.