sensemaking 0.19.0 → 0.19.2

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is\n// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,oIAAoI;AACpI,wOAAwO;AACxO,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["SenseError","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","duckdbDialect","beginMode","checkColumnLimit","count","columnTypeSuffix"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAK7C,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;AACxE;AAEA,OAAO,MAAMiB,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQb,yBAAyB;YACnC,MAAM,IAAIT,WAAW,gBAAgB,CAAC,uBAAuB,EAAEsB,MAAM,8CAA8C,EAAEb,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fc,kBAAkB,IAAM;IACxBb;AACF,EAAE"}

package/dist/esm/store/duckdb/store.js

@@ -1,18 +1,20 @@
 import { STORE_DIMS } from '../../embed/types.js';
+import { reconcile } from '../reconcile.js';
 import { getColumns } from '../shared.js';
 import { withTransaction } from '../transaction.js';
 import { hasVectorRow, pendingRows } from '../vectors.js';
 import { storeRowToJs } from './connection.js';
 import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
-import { reconcile } from './reconcile.js';
+import { duckdbDialect } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
 // Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native
 // FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.
 export const CAPABILITIES = new Set([
     'lexical',
     'phrases',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
 // Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.
 // The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.
@@ -34,7 +36,7 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
             return withTransaction(conn, fn);
         },
         async reconcile () {
-            const result = await reconcile(conn, cfg, baseDir);
+            const result = await reconcile(conn, cfg, baseDir, duckdbDialect);
             // content may have changed; the fts index is rebuilt lazily, on the next lexical query
             // that needs it, not here (see lexical.ts's FtsIndexState).
             lex.markStale();

package/dist/esm/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMjB,YAAYwC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI1C,YAAY4C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMjB,YAAY+C,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir, duckdbDialect);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","reconcile","getColumns","withTransaction","hasVectorRow","pendingRows","storeRowToJs","fieldStats","createLexicalIndex","duckdbDialect","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","scopeWhere","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,EAAEC,WAAW,QAAQ,gBAAgB;AAC1D,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,oIAAoI;AACpI,8NAA8N;AAC9N,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB,EAAE;AAEjH,8JAA8J;AAC9J,yKAAyK;AACzK,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAO3B,gBAAgBe,MAAMY;QAC/B;QACA,MAAM7B;YACJ,MAAM8B,SAAS,MAAM9B,UAAUiB,MAAMC,KAAKC,SAASX;YACnD,uFAAuF;YACvF,4DAA4D;YAC5DY,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAMhC,WAAWgB;iBAAO;YACtC;YACAX,YAAY,CAAC2B,SAASC,aAAe5B,WAAWW,MAAMgB,SAASC;QACjE;QACAC,SAAS;YACPC,OAAOhB,IAAIgB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,IAAMlC,YAAYa;YAC3BsB,cAAc,CAACC,OAAS7B,iBAAiBK,QAAQC,MAAMlB,YAAYyC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYpC,eAAeO,QAAQ0B,IAAI3C,YAAY6C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAAStC,YAAYM,QAAQC,MAAMlB,YAAYgD,MAAMC;YACrEC,WAAW,CAACF,OAAS5C,aAAac,MAAM8B;QAC1C;QACA,uCAAuC;QACvC,MAAMG;YACJ,OAAO,CAAC;QACV;QACAC,KAAK;YACH,MAAM1B,SAAQD,GAAW;gBACvB,MAAM4B,WAAW,MAAMpC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMoB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEnC,MAAM+B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAM1B,OAAOsB,OAAOK,eAAe;4BACnC,MAAOH,UAAUxB,KAAKoB,MAAM,EAAEI,UAAW,MAAM3D,aAAamC,IAAI,CAACwB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFrD,OAAOsD,cAAc;YACrBvD,SAASwD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/index.d.ts

@@ -7,3 +7,4 @@ export { getMeta, setMeta } from './shared.js';
 export type { OpenResult } from './sqlite/open.js';
 export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
+export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js

@@ -50,3 +50,4 @@ export function requireWatchConcurrency(cfg) {
 export { clearCache } from './cache.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export { CAPABILITY_NAMES } from './types.js';

package/dist/esm/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","openTurso","TURSO_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","turso","entryFor","name","entry","Object","keys","join","openStore","cfg","has","requireWatchConcurrency","clearCache","getMeta","setMeta","DB_FILENAME","docCount","SCHEMA_VERSION","CAPABILITY_NAMES"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AACxE,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASJ,gBAAgBK,kBAAkB,QAAQ,mBAAmB;AAQtE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcL;QAAqBM,MAAMP;IAAW;IAC9DQ,QAAQ;QAAEF,cAAcP;QAAqBQ,MAAMV;IAAW;IAC9DY,OAAO;QAAEH,cAAcH;QAAoBI,MAAML;IAAU;AAC7D;AAEA,SAASQ,SAASC,IAAe;IAC/B,MAAMC,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIhB,WAAW,iBAAiB,CAAC,uBAAuB,EAAEe,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IACnI,OAAOH;AACT;AAEA,OAAO,eAAeI,UAAUC,GAAmB;IACjD,MAAMN,OAAOhB,UAAUsB;IACvB,MAAML,QAAQF,SAASC;IAEvB,8FAA8F;IAC9F,oJAAoJ;IACpJ,IAAIjB,gBAAgBuB,QAAQ,CAACL,MAAMN,YAAY,CAACY,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACU;AACpB;AAEA,wFAAwF;AACxF,iFAAiF;AACjF,OAAO,SAASE,wBAAwBF,GAAmB;IACzD,MAAMN,OAAOhB,UAAUsB;IACvB,IAAI,CAACP,SAASC,MAAML,YAAY,CAACY,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAItB,WAAW,4BAA4B,CAAC,OAAO,EAAEe,KAAK,qPAAqP,CAAC;IACxT;AACF;AAEA,SAASS,UAAU,QAAQ,aAAa;AACxC,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,SAASC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB;AAEzE,SAASC,gBAAgB,QAAQ,aAAa"}

package/dist/esm/store/reconcile.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;

package/dist/esm/store/reconcile.js

@@ -0,0 +1,130 @@
+import { activeFeatures } from '../features/index.js';
+import { progress } from '../output/progress.js';
+import { listFiles, RESERVED_COLUMNS } from '../scan/index.js';
+import { reparseFiles } from '../scan/reparse.js';
+import { getColumns, quoteIdent } from './shared.js';
+import { withTransaction } from './transaction.js';
+// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect
+// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,
+// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks
+// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter
+// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.
+// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
+// computed value on every touch, not just the reconciles that recompute it.
+const CORE_FRONTMATTER_COLUMNS = new Set([
+    'path',
+    '_mtime',
+    '_ctime',
+    '_size',
+    '_parse_error'
+]);
+export async function reconcile(conn, cfg, baseDir, dialect) {
+    const files = listFiles(cfg, baseDir);
+    const currentSet = new Set(files.map((f)=>f.relPath));
+    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
+    const existingRows = await existingStmt.all();
+    const existing = new Map(existingRows.map((r)=>[
+            r.path,
+            r
+        ]));
+    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
+    const toReparse = files.filter((f)=>{
+        const row = existing.get(f.relPath);
+        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
+    });
+    if (vanished.length === 0 && toReparse.length === 0) return {
+        parsed: 0,
+        warnings: []
+    };
+    const features = activeFeatures(cfg);
+    const seenColumns = await getColumns(conn);
+    // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short
+    // reconciles stay silent (progress() has a threshold).
+    const report = progress('reparsing files', toReparse.length);
+    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
+    report.finish();
+    for (const col of newColumns)seenColumns.add(col);
+    const allColumns = [
+        ...seenColumns
+    ];
+    // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific
+    // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.
+    dialect.checkColumnLimit(allColumns.length);
+    // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a
+    // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).
+    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
+    // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --
+    // sqlite's content rows are coupled to that rowid.
+    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
+    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
+    const delta = {
+        files,
+        reparsed: parsedDocs.map((d)=>d.relPath),
+        added,
+        vanished
+    };
+    const addedSet = new Set(added);
+    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
+    // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to
+    // be reinserted fresh. Disjoint from `added`, which has nothing to clear.
+    const touched = [
+        ...vanished,
+        ...reparsedExisting
+    ];
+    const txStart = Date.now();
+    await withTransaction(conn, async ()=>{
+        var _feature_remove, _feature_afterReconcile;
+        // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
+        // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
+        const present = await getColumns(conn);
+        for (const col of newColumns)if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);
+        if (parsedDocs.length > 0) {
+            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
+                    var _doc_data_col;
+                    if (col === 'path') return doc.relPath;
+                    if (col === '_mtime') return doc.mtimeMs;
+                    if (col === '_ctime') return doc.ctimeMs;
+                    if (col === '_size') return doc.size;
+                    // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.
+                    if (col === '_parse_error') return doc.parseError;
+                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
+                }));
+            await conn.runBatch(insertSql, rows);
+        }
+        // After the upsert, so every doc already has the frontmatter row sqlite's content rowid
+        // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.
+        await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);
+        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
+        // new docs have nothing to clear, which keeps cold builds linear.
+        if (touched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', touched.map((p)=>[
+                p
+            ]));
+        const presetRows = [];
+        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
+            doc.relPath,
+            presetName
+        ]);
+        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
+        // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's
+        // node set, so a lingering vanished row would dilute rank mass across every surviving note.
+        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
+                p
+            ]));
+        if (touched.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, touched, delta));
+        for (const feature of features){
+            var _feature_store;
+            const docsForFeature = parsedDocs.map((doc)=>({
+                    path: doc.relPath,
+                    extracted: doc.extracted[feature.name]
+                }));
+            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
+        }
+        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
+    }, dialect.beginMode());
+    const durationMs = Date.now() - txStart;
+    if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);
+    return {
+        parsed: parsedDocs.length,
+        warnings
+    };
+}

package/dist/esm/store/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/reconcile.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../output/progress.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../scan/index.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport { getColumns, quoteIdent } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// One reconcile algorithm shared by every store, parameterised by a per-engine ReconcileDialect\n// (types.ts). Ordering is universal, not a dialect concern: ALTER, then the frontmatter upsert,\n// then reconcileContent, then preset_files, then the vanished-frontmatter delete, then feature hooks\n// -- a vanished path's content delete (inside reconcileContent) must precede its frontmatter\n// delete, since sqlite's delete SQL resolves the row via its frontmatter rowid.\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: a store's own failure past this point is a raw, engine-specific\n  // error with no indication of the boundary or the levers -- dialect.checkColumnLimit names both.\n  dialect.checkColumnLimit(allColumns.length);\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT DO UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // sqlite's content rows are coupled to that rowid.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n  // Paths whose content (and, per feature, other rows) need clearing: gone entirely, or about to\n  // be reinserted fresh. Disjoint from `added`, which has nothing to clear.\n  const touched = [...vanished, ...reparsedExisting];\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}${dialect.columnTypeSuffix()}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // After the upsert, so every doc already has the frontmatter row sqlite's content rowid\n      // couples to. ON CONFLICT DO UPDATE preserves that rowid, so a reparse keeps its identity.\n      await dialect.reconcileContent(conn, touched, parsedDocs, delta, cfg);\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      if (touched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          touched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      // Before the feature hooks, never after: rank's afterReconcile reads frontmatter as PageRank's\n      // node set, so a lingering vanished row would dilute rank mass across every surviving note.\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      if (touched.length > 0) for (const feature of features) await feature.remove?.(conn, touched, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n\n  const durationMs = Date.now() - txStart;\n  if (dialect.recordDuration) await dialect.recordDuration(conn, durationMs);\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","reconcile","conn","cfg","baseDir","dialect","files","currentSet","map","f","relPath","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","checkColumnLimit","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","touched","txStart","Date","now","feature","present","exec","columnTypeSuffix","rows","doc","ctimeMs","parseError","data","runBatch","reconcileContent","presetRows","presetName","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","beginMode","durationMs","recordDuration"],"mappings":"AACA,SAASA,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,mBAAmB;AAC/D,SAASC,YAAY,QAAQ,qBAAqB;AAClD,SAASC,UAAU,EAAEC,UAAU,QAAQ,cAAc;AACrD,SAASC,eAAe,QAAQ,mBAAmB;AAGnD,gGAAgG;AAChG,gGAAgG;AAChG,qGAAqG;AACrG,6FAA6F;AAC7F,gFAAgF;AAEhF,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACvG,MAAMC,QAAQb,UAAUU,KAAKC;IAC7B,MAAMG,aAAa,IAAIP,IAAIM,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IAErD,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaL,GAAG,CAAC,CAACS,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACV,WAAWc,GAAG,CAACJ,EAAEC,IAAI,GAAGV,GAAG,CAAC,CAACS,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYhB,MAAMc,MAAM,CAAC,CAACX;QAC9B,MAAMc,MAAMR,SAASS,GAAG,CAACf,EAAEC,OAAO;QAClC,OAAO,CAACa,OAAOA,IAAIE,MAAM,KAAKhB,EAAEiB,OAAO,IAAIH,IAAII,KAAK,KAAKlB,EAAEmB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWzC,eAAeY;IAChC,MAAM8B,cAAc,MAAMrC,WAAWM;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAMgC,SAAS1C,SAAS,mBAAmB8B,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAM1C,aAAa2B,WAAWU,UAAU7B,KAAK8B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,yFAAyF;IACzF,iGAAiG;IACjG5B,QAAQsC,gBAAgB,CAACD,WAAWb,MAAM;IAC1C,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMe,kBAAkBF,WAAWtB,MAAM,CAAC,CAACyB,IAAM9C,yBAAyBsB,GAAG,CAACwB,MAAM,CAACnD,iBAAiB2B,GAAG,CAACwB;IAC1G,yFAAyF;IACzF,mDAAmD;IACnD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBpC,GAAG,CAACX,YAAYkD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBpC,GAAG,CAAC,IAAM,KAAKuC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLxB,MAAM,CAAC,CAACyB,IAAMA,MAAM,QACpBrC,GAAG,CAAC,CAACqC,IAAM,GAAGhD,WAAWgD,GAAG,YAAY,EAAEhD,WAAWgD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQ1B,UAAUF,MAAM,CAAC,CAACX,IAAM,CAACM,SAASM,GAAG,CAACZ,EAAEC,OAAO,GAAGF,GAAG,CAAC,CAACC,IAAMA,EAAEC,OAAO;IACpF,MAAMuC,QAAwB;QAAE3C;QAAO4C,UAAUd,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO;QAAGsC;QAAO7B;IAAS;IACnG,MAAMiC,WAAW,IAAIpD,IAAIgD;IACzB,MAAMK,mBAAmBjB,WAAW5B,GAAG,CAAC,CAAC2C,IAAMA,EAAEzC,OAAO,EAAEU,MAAM,CAAC,CAACkC,IAAM,CAACF,SAAS/B,GAAG,CAACiC;IACtF,+FAA+F;IAC/F,0EAA0E;IAC1E,MAAMC,UAAU;WAAIpC;WAAakC;KAAiB;IAElD,MAAMG,UAAUC,KAAKC,GAAG;IACxB,MAAM5D,gBACJI,MACA;YA4CgEyD,iBAKxBA;QAhDtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMhE,WAAWM;QACjC,KAAK,MAAMsC,OAAOH,WAAY,IAAI,CAACuB,QAAQvC,GAAG,CAACmB,MAAM,MAAMtC,KAAK2D,IAAI,CAAC,CAAC,mCAAmC,EAAEhE,WAAW2C,OAAOnC,QAAQyD,gBAAgB,IAAI;QAEzJ,IAAI1B,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMkC,OAAO3B,WAAW5B,GAAG,CAAC,CAACwD,MAC3BpB,gBAAgBpC,GAAG,CAAC,CAACgC;wBAOZwB;oBANP,IAAIxB,QAAQ,QAAQ,OAAOwB,IAAItD,OAAO;oBACtC,IAAI8B,QAAQ,UAAU,OAAOwB,IAAItC,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAOwB,IAAIC,OAAO;oBACxC,IAAIzB,QAAQ,SAAS,OAAOwB,IAAIpC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAOwB,IAAIE,UAAU;oBACjD,QAAOF,gBAAAA,IAAIG,IAAI,CAAC3B,IAAI,cAAbwB,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAM9D,KAAKkE,QAAQ,CAACtB,WAAWiB;QACjC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAM1D,QAAQgE,gBAAgB,CAACnE,MAAMqD,SAASnB,YAAYa,OAAO9C;QAEjE,0FAA0F;QAC1F,kEAAkE;QAClE,IAAIoD,QAAQ1B,MAAM,GAAG,GACnB,MAAM3B,KAAKkE,QAAQ,CACjB,6CACAb,QAAQ/C,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAE1B,MAAMgB,aAA0B,EAAE;QAClC,KAAK,MAAMN,OAAO5B,WAAY,KAAK,MAAMmC,cAAcP,IAAIQ,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACT,IAAItD,OAAO;YAAE6D;SAAW;QAC7G,IAAID,WAAWzC,MAAM,GAAG,GAAG,MAAM3B,KAAKkE,QAAQ,CAAC,2DAA2DE;QAE1G,+FAA+F;QAC/F,4FAA4F;QAC5F,IAAInD,SAASU,MAAM,GAAG,GACpB,MAAM3B,KAAKkE,QAAQ,CACjB,4CACAjD,SAASX,GAAG,CAAC,CAAC8C,IAAM;gBAACA;aAAE;QAG3B,IAAIC,QAAQ1B,MAAM,GAAG,GAAG,KAAK,MAAM8B,WAAW3B,SAAU,QAAM2B,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBzD,MAAMqD,SAASN;QAC9F,KAAK,MAAMU,WAAW3B,SAAU;gBAExB2B;YADN,MAAMgB,iBAAiCvC,WAAW5B,GAAG,CAAC,CAACwD,MAAS,CAAA;oBAAE9C,MAAM8C,IAAItD,OAAO;oBAAEkE,WAAWZ,IAAIY,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBzD,MAAMyE,gBAAgB1B;QAC9C;QACA,KAAK,MAAMU,WAAW3B,SAAU,QAAM2B,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyBzD,MAAM+C;IACvE,GACA5C,QAAQ2E,SAAS;IAGnB,MAAMC,aAAaxB,KAAKC,GAAG,KAAKF;IAChC,IAAInD,QAAQ6E,cAAc,EAAE,MAAM7E,QAAQ6E,cAAc,CAAChF,MAAM+E;IAE/D,OAAO;QAAEnD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/shared.d.ts

@@ -3,3 +3,4 @@ export declare function quoteIdent(name: string): string;
 export declare function getColumns(conn: Connection): Promise<Set<string>>;
 export declare function getMeta(conn: Connection, key: string): Promise<string | null>;
 export declare function setMeta(conn: Connection, key: string, value: string | null): Promise<void>;
+export declare function recordReconcileDuration(conn: Connection, ms: number): Promise<void>;

package/dist/esm/store/shared.js

@@ -22,3 +22,12 @@ export async function setMeta(conn, key, value) {
     const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
     await stmt.run(key, value);
 }
+// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the
+// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.
+export async function recordReconcileDuration(conn, ms) {
+    const prevRaw = await getMeta(conn, 'reconcile_max_ms');
+    // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)
+    // still gets recorded instead of losing to the "nothing recorded yet" default.
+    const prevMax = prevRaw === null ? -1 : Number(prevRaw);
+    if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));
+}

package/dist/esm/store/shared.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n"],"names":["quoteIdent","name","split","join","getColumns","conn","stmt","prepare","rows","all","Set","map","r","getMeta","key","row","get","value","setMeta","run"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,OAAO,SAASA,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWC,IAAgB;IAC/C,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAO,IAAIC,IAAIF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;AACvC;AAEA,OAAO,eAAeY,QAAQR,IAAgB,EAAES,GAAW;IACzD,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMQ,MAAO,MAAMT,KAAKU,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQb,IAAgB,EAAES,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMX,OAAO,MAAMD,KAAKE,OAAO,CAAC;QAChC,MAAMD,KAAKa,GAAG,CAACL;QACf;IACF;IACA,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMD,KAAKa,GAAG,CAACL,KAAKG;AACtB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n\n// Reconcile's own write-transaction duration, for open()'s derived busy_timeout: keep the\n// observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\nexport async function recordReconcileDuration(conn: Connection, ms: number): Promise<void> {\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (ms > prevMax) await setMeta(conn, 'reconcile_max_ms', String(ms));\n}\n"],"names":["quoteIdent","name","split","join","getColumns","conn","stmt","prepare","rows","all","Set","map","r","getMeta","key","row","get","value","setMeta","run","recordReconcileDuration","ms","prevRaw","prevMax","Number","String"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,OAAO,SAASA,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWC,IAAgB;IAC/C,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAO,IAAIC,IAAIF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;AACvC;AAEA,OAAO,eAAeY,QAAQR,IAAgB,EAAES,GAAW;IACzD,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMQ,MAAO,MAAMT,KAAKU,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQb,IAAgB,EAAES,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMX,OAAO,MAAMD,KAAKE,OAAO,CAAC;QAChC,MAAMD,KAAKa,GAAG,CAACL;QACf;IACF;IACA,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMD,KAAKa,GAAG,CAACL,KAAKG;AACtB;AAEA,0FAA0F;AAC1F,qGAAqG;AACrG,OAAO,eAAeG,wBAAwBf,IAAgB,EAAEgB,EAAU;IACxE,MAAMC,UAAU,MAAMT,QAAQR,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAMkB,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,KAAKE,SAAS,MAAML,QAAQb,MAAM,oBAAoBoB,OAAOJ;AACnE"}

package/dist/esm/store/sqlite/connection.js

@@ -1,4 +1,4 @@
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 // Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real
 // work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.
 let SqliteStatement = class SqliteStatement {
@@ -39,7 +39,7 @@ export function createConnection(db) {
             await withTransaction(conn, async ()=>{
                 const stmt = db.prepare(sql);
                 for (const row of paramRows)stmt.run(...row);
-            });
+            }, BEGIN_WRITE);
         }
     };
     return conn;

package/dist/esm/store/sqlite/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const stmt = db.prepare(sql);\n        for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAGpD,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBAAgBa,MAAM;gBAC1B,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C;QACF;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = db.prepare(sql);\n          for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAGjE,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBACJa,MACA;gBACE,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C,GACArB;QAEJ;IACF;IACA,OAAOc;AACT"}

package/dist/esm/store/sqlite/open.js

@@ -8,10 +8,12 @@ import { rekeyChunkText } from '../../embed/handoff.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { clearCache } from '../cache.js';
+import { reconcile } from '../reconcile.js';
 import { getMeta, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
-import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
+import { sqliteDialect } from './reconcile.js';
+import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, signatureDiff } from './signature.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.db';
@@ -64,6 +66,20 @@ async function ensureSchema(conn, cfg, tokenize) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
+// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY
+// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.
+function setJournalWal(db) {
+    const deadline = Date.now() + 5000;
+    for(;;){
+        try {
+            db.exec('PRAGMA journal_mode = WAL');
+            return;
+        } catch (err) {
+            if (Date.now() >= deadline || !/database is locked|busy/i.test(err.message)) throw err;
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
+        }
+    }
+}
 async function connect(cfg) {
     const stateDir = join(cfg.baseDir, STATE_DIR);
     mkdirSync(stateDir, {
@@ -76,10 +92,12 @@ async function connect(cfg) {
     let closed = false;
     try {
         var _ref;
-        db.exec('PRAGMA journal_mode = WAL');
-        // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
-        // that outwaits it still fails loudly.
+        // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
+        // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
         db.exec('PRAGMA busy_timeout = 30000');
+        // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy
+        // handler for it, so a concurrent cold open needs its own bounded wait.
+        setJournalWal(db);
         registerFunctions(db, contentTokenize(cfg) === undefined);
         const conn = createConnection(db);
         db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
@@ -112,7 +130,7 @@ async function connect(cfg) {
                 await withTransaction(conn, async ()=>{
                     await conn.exec('DROP TABLE IF EXISTS content');
                     await createContentTable(conn, tokenize);
-                });
+                }, BEGIN_WRITE);
                 tokenizeOnlyRebuild = true;
             } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
                 // First sight of a resolved weight identity: the model itself hasn't changed, so
@@ -138,7 +156,9 @@ async function connect(cfg) {
             clearCache(cfg);
             return connect(cfg);
         }
-        await ensureSchema(conn, cfg, tokenize);
+        // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
+        // here both see it missing and the second ALTER fails with a duplicate column.
+        await withTransaction(conn, ()=>ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);
         let rebuildWarnings = [];
         if (tokenizeOnlyRebuild) {
             rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);
@@ -148,7 +168,7 @@ async function connect(cfg) {
         // Installed before reconcile() -- that call is the one that races a watcher's transaction.
         const recordedMaxMs = Number((_ref = await getMeta(conn, 'reconcile_max_ms')) !== null && _ref !== void 0 ? _ref : '0');
         db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600000)}`);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);
+        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, sqliteDialect);
         return {
             db,
             conn,

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(conn, async () => {\n          await conn.exec('DROP TABLE IF EXISTS content');\n          await createContentTable(conn, tokenize);\n        });\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","rekeyChunkText","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","withTransaction","createConnection","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAazB,gBAAgBwB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI1B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEqB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWtC,eAAemB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMhC,QAAQgC,MAAM,sBAAuB,MAAM,MAAM/B,QAAQ+B,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMZ,QAAQgC,MAAM,gBAAiB,MAAM,MAAM/B,QAAQ+B,MAAM,YAAYvC,iBAAiBuB,KAAKlB;AACxG;AAEA,eAAeuC,QAAQrB,GAAmB;IACxC,MAAMsB,WAAWhD,KAAK0B,IAAIuB,OAAO,EAAE7C;IACnCL,UAAUiD,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASnD,KAAKgD,UAAU3B;IAE9B,MAAMI,KAAK,IAAIxB,aAAakD;IAC5B,6FAA6F;IAC7F,wFAAwF;IACxF,IAAIC,SAAS;IACb,IAAI;YA8E4B;QA7E9B3B,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,uCAAuC;QACvCN,GAAGM,IAAI,CAAC;QACRZ,kBAAkBM,IAAIvB,gBAAgBwB,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAM2B,UAAU,MAAM3C,QAAQgC,MAAM;QACpC,MAAMY,WAAW,MAAM5C,QAAQgC,MAAM;QACrC,MAAMa,eAAepD,iBAAiBuB,KAAKlB;QAC3C,IAAIgD,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY/B,kBAAoBgC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAY/B,gBAAgB;gBAClDmC,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACT3B,GAAGkC,KAAK;gBACRlD,WAAWiB;gBACX,OAAOqB,QAAQrB;YACjB;YACA,MAAMkC,cAAc9C,qBAAqBwC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,6FAA6F;YAC7F,2FAA2F;YAC3F,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDL,QAAQC,KAAK,CAAC;gBACd,2FAA2F;gBAC3F,yFAAyF;gBACzF,MAAM9C,gBAAgB8B,MAAM;oBAC1B,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC;gBACAa,sBAAsB;YACxB,OAAO,IAAII,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAY/C,qBAAqBuC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAM/C,QAAQ+B,MAAM,YAAYa;YAClC,OAAO;gBACL,MAAMQ,UAAU7C,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEK,QAAQ,oBAAoB,CAAC;gBACpEX,SAAS;gBACT3B,GAAGkC,KAAK;gBACRlD,WAAWiB;gBACX,OAAOqB,QAAQrB;YACjB;QACF;QAEA,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAMsC,SAAS9B,eAAeT;QAC9B,IAAIuC,WAAW,QAAQA,WAAWrB,UAAU;YAC1Cc,QAAQC,KAAK,CAAC;YACdN,SAAS;YACT3B,GAAGkC,KAAK;YACRlD,WAAWiB;YACX,OAAOqB,QAAQrB;QACjB;QAEA,MAAMkB,aAAaF,MAAMhB,KAAKiB;QAE9B,IAAIsB,kBAA4B,EAAE;QAClC,IAAIT,qBAAqB;YACvBS,kBAAkB,MAAMjD,oBAAoB0B,MAAMhB,KAAKA,IAAIuB,OAAO;YAClE,MAAMtC,QAAQ+B,MAAM,YAAYa;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMW,gBAAgBC,QAAQ,OAAA,MAAMzD,QAAQgC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMvD,UAAUyB,MAAMhB,KAAKA,IAAIuB,OAAO;QAEnE,OAAO;YAAExB;YAAIiB;YAAMhB;YAAKyB;YAAQoB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOxC,KAAK;QACZ,IAAI,CAACoB,QAAQ3B,GAAGkC,KAAK;QACrB,MAAM3B;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAeyC,WAAW/C,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKgD,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQrB;IAC/E,MAAMiD,QAAQvD,YAAYK,IAAIiB,MAAMgC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F5C,eAAeqC,MAAMiC;IACrB,OAAO;QAAEA,OAAOA;QAAOjD,KAAKgD;QAAavB;QAAQoB;QAAQC;IAAS;AACpE;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMvC,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMyC,KAAKxC,GAAG,EAAC,EAAqByC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { sqliteDialect } from './reconcile.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, signatureDiff } from './signature.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(\n          conn,\n          async () => {\n            await conn.exec('DROP TABLE IF EXISTS content');\n            await createContentTable(conn, tokenize);\n          },\n          BEGIN_WRITE\n        );\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n    // here both see it missing and the second ALTER fails with a duplicate column.\n    await withTransaction(conn, () => ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, sqliteDialect);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","rekeyChunkText","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","sqliteDialect","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,aAAa,QAAQ,iBAAiB;AAChH,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAa3B,gBAAgB0B;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI5B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEuB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWxC,eAAeqB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMjC,QAAQiC,MAAM,sBAAuB,MAAM,MAAMhC,QAAQgC,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMb,QAAQiC,MAAM,gBAAiB,MAAM,MAAMhC,QAAQgC,MAAM,YAAYzC,iBAAiByB,KAAKpB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASyC,cAActB,EAAgB;IACrC,MAAMuB,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFzB,GAAGM,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIiB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACnB,IAAcC,OAAO,GAAG,MAAMD;YAC9FoB,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQ9B,GAAmB;IACxC,MAAM+B,WAAW3D,KAAK4B,IAAIgC,OAAO,EAAExD;IACnCL,UAAU4D,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS9D,KAAK2D,UAAUpC;IAE9B,MAAMI,KAAK,IAAI1B,aAAa6D;IAC5B,6FAA6F;IAC7F,wFAAwF;IACxF,IAAIC,SAAS;IACb,IAAI;YAsF4B;QArF9B,6FAA6F;QAC7F,yFAAyF;QACzFpC,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEgB,cAActB;QACdN,kBAAkBM,IAAIzB,gBAAgB0B,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAMoC,UAAU,MAAMrD,QAAQiC,MAAM;QACpC,MAAMqB,WAAW,MAAMtD,QAAQiC,MAAM;QACrC,MAAMsB,eAAe/D,iBAAiByB,KAAKpB;QAC3C,IAAI2D,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAYxC,kBAAoByC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYxC,gBAAgB;gBAClD4C,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACTpC,GAAG2C,KAAK;gBACR7D,WAAWmB;gBACX,OAAO8B,QAAQ9B;YACjB;YACA,MAAM2C,cAActD,qBAAqBgD,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,6FAA6F;YAC7F,2FAA2F;YAC3F,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDL,QAAQC,KAAK,CAAC;gBACd,2FAA2F;gBAC3F,yFAAyF;gBACzF,MAAMvD,gBACJ8B,MACA;oBACE,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC,GACAhC;gBAEFsD,sBAAsB;YACxB,OAAO,IAAII,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAYvD,qBAAqB+C,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAMzD,QAAQgC,MAAM,YAAYsB;YAClC,OAAO;gBACL,MAAMQ,UAAUtD,cAAc6C,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEK,QAAQ,oBAAoB,CAAC;gBACpEX,SAAS;gBACTpC,GAAG2C,KAAK;gBACR7D,WAAWmB;gBACX,OAAO8B,QAAQ9B;YACjB;QACF;QAEA,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM+C,SAASvC,eAAeT;QAC9B,IAAIgD,WAAW,QAAQA,WAAW9B,UAAU;YAC1CuB,QAAQC,KAAK,CAAC;YACdN,SAAS;YACTpC,GAAG2C,KAAK;YACR7D,WAAWmB;YACX,OAAO8B,QAAQ9B;QACjB;QAEA,+FAA+F;QAC/F,+EAA+E;QAC/E,MAAMd,gBAAgB8B,MAAM,IAAME,aAAaF,MAAMhB,KAAKiB,WAAWhC;QAErE,IAAI+D,kBAA4B,EAAE;QAClC,IAAIT,qBAAqB;YACvBS,kBAAkB,MAAMzD,oBAAoByB,MAAMhB,KAAKA,IAAIgC,OAAO;YAClE,MAAMhD,QAAQgC,MAAM,YAAYsB;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMW,gBAAgBC,QAAQ,OAAA,MAAMnE,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAE8C,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzE,UAAUkC,MAAMhB,KAAKA,IAAIgC,OAAO,EAAE5C;QAErE,OAAO;YAAEW;YAAIiB;YAAMhB;YAAKkC;YAAQoB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOjD,KAAK;QACZ,IAAI,CAAC6B,QAAQpC,GAAG2C,KAAK;QACrB,MAAMpC;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAekD,WAAWxD,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKyD,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQ9B;IAC/E,MAAM0D,QAAQhE,YAAYK,IAAIiB,MAAMyC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1FvD,eAAeuC,MAAM0C;IACrB,OAAO;QAAEA,OAAOA;QAAO1D,KAAKyD;QAAavB;QAAQoB;QAAQC;IAAS;AACpE;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMhD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMkD,KAAKjD,GAAG,EAAC,EAAqBkD,CAAC;AAChD"}