sensemaking 0.19.0 → 0.19.1

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping\n// (open.ts's busy timeout derives from it). content is a plain table with \"_ngram\" sidecars for lexical.ts.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed\n  // max is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // gets recorded rather than losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;yBACE;wBAC0B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,8FAA8F;AAC9F,4GAA4G;AAC5G,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAEhC,yFAAyF;AACzF,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEO,SAAeJ,UAAUc,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YASAC,iBAEAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA0DAC,YACAC,SAGAC;;;;oBA5GAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAEhC;;wBAAMC,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAE/B,OAAO;wBAClC,OAAO,CAACuC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,uBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAW2B,MAAM,GAAGpD,yBAAyB;wBAC/C,MAAM,IAAI8D,oBAAU,CAClB,gBACA,AAAC,0BAAwG9D,OAA/EyB,WAAW2B,MAAM,EAAC,+DAAqF,OAAxBpD,yBAAwB;oBAErI;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE0B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sHAAsH;oBAChHpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAE/B,OAAO;uBAAG8B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAC9EsB,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAE7D,OAAO;;wBAAGqB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAE7D,OAAO;uBAAEqC,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCA+CyCiE,iBAK7BA,yBAnDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAGHkD,MAgBFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM1E,KAAwB,YAAA,QAAM2E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAnDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKwE,IAAI,CAAC,AAAC,sCAAqD,OAAhBf,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAAClC;uDAC3BuB,gBAAgBW,GAAG,CAAC,SAACb;wDAOZrB;oDANP,IAAIqB,QAAQ,QAAQ,OAAOrB,IAAII,OAAO;oDACtC,IAAIiB,QAAQ,UAAU,OAAOrB,IAAI8C,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOrB,IAAI8E,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOrB,IAAIgD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOrB,IAAI+E,UAAU;oDACjD,QAAO/E,gBAAAA,IAAIgF,IAAI,CAAC3D,IAAI,cAAbrB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMK,KAAK4E,QAAQ,CAACzD,WAAW+C;;;4CAA/B;;;4CAGF,0FAA0F;4CAC1F,8DAA8D;4CACxDC,iBAAiB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACpC4C,CAAAA,eAAevB,MAAM,GAAG,CAAA,GAAxBuB;;;;4CACF;;gDAAMnE,KAAK4E,QAAQ,CACjB,wCACAT,eAAetC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CAAuB;;gDAAMb,KAAK4E,QAAQ,CAACnF,oBAAoBoB,WAAWgB,GAAG,CAACnC;;;4CAAvD;;;iDAEvBc,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAK4E,QAAQ,CACjB,4CACApE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DO,gBAAgB,AAAC,qBAAG5D,iBAAU,qBAAGe;iDACnC6C,CAAAA,cAAcxB,MAAM,GAAG,CAAA,GAAvBwB;;;;4CACF;;gDAAMpE,KAAK4E,QAAQ,CACjB,6CACAR,cAAcvC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIQ;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAaxD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMlB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIkF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAEnF,IAAII,OAAO;gEAAEuE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWzB,MAAM,GAAG,CAAA,GAApByB;;;;4CAAuB;;gDAAMrE,KAAK4E,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG/D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BgD,CAAAA,aAAa3B,MAAM,GAAG,CAAA,GAAtB2B;;;;;;;;;;;;4CAA8B,aAAiB7D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBjE,MAAMuE,cAAclD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiCnE,WAAWgB,GAAG,CAAC,SAAClC;uEAAS;oEAAEwC,MAAMxC,IAAII,OAAO;oEAAEkF,WAAWtF,IAAIsF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBjE,MAAMgF,gBAAgB3D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBArDA;oBAuDA,8FAA8F;oBAC9F,qDAAqD;oBAC/CI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM6D,IAAAA,iBAAO,EAACrF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,0EAA0E;oBACpEC,UAAUD,YAAY,OAAO,CAAC,IAAI4D,OAAO5D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM8D,IAAAA,iBAAO,EAACvF,MAAM,oBAAoBwF,OAAO/D;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping\n// (open.ts's busy timeout derives from it). content is a plain table with \"_ngram\" sidecars for lexical.ts.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row (see reconcile).\nconst FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n      }\n\n      // Tantivy maintains the FTS index per inserted row and the cost grows with the batch, so a\n      // large insert is superlinear: 6,566 notes took 945s with the index live, 3.3s built after.\n      // Measured crossover against a full rebuild, in changed files: ~250 on a 6,566-note tree\n      // (100 -> 895ms, 300 -> 4.1s, 1,300 -> 45.5s, rebuild 3.35s), ~295 on 13,132 (250 -> 5.2s,\n      // 400 -> 10.0s, rebuild 6.7s). Doubling the corpus moved it 250 to 295, so a constant fits\n      // where a ratio does not: 20% would allow 1,313 files here, costing 45s to save 3.3s.\n      // Erring low is the cheap direction, since a needless rebuild is linear and predictable.\n      const churn = delta.reparsed.length + delta.vanished.length;\n      const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n      if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n      // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n      // content), so vanished and reparsed docs delete in one pass.\n      const contentTouched = [...vanished, ...reparsedExisting];\n      if (contentTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM content WHERE \"path\" = ?',\n          contentTouched.map((p) => [p])\n        );\n      if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n      if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      const presetTouched = [...vanished, ...reparsedExisting];\n      if (presetTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          presetTouched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      const removedPaths = [...vanished, ...reparsedExisting];\n      if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    BEGIN_WRITE\n  );\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed\n  // max is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // gets recorded rather than losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","FTS_REBUILD_THRESHOLD","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","churn","bulk","name","contentTouched","ddl","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","CONTENT_FTS_NAMES","CONTENT_FTS_DDL","presets","push","remove","docsForFeature","extracted","store","afterReconcile","BEGIN_WRITE","getMeta","Number","setMeta","String"],"mappings":";;;;+BAuCsBA;;;eAAAA;;;wBAtCK;uBACI;0BAEN;wBAEmB;yBACf;yBACE;wBAC0B;6BACZ;sBAEM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEnD,8FAA8F;AAC9F,4GAA4G;AAC5G,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAEhC,mGAAmG;AACnG,IAAMC,wBAAwB;AAE9B,yFAAyF;AACzF,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAAST,OAASM,YAATN;IACxB,OAAO;QAACM,IAAII,OAAO;QAAEF;QAAOC;QAAST;QAAMD,aAAaS;QAAQT,aAAaU;QAAUV,aAAaC;KAAM;AAC5G;AAEO,SAAeJ,UAAUe,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YASAC,iBAEAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA0EAC,YACAC,SAGAC;;;;oBA5HAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIjB,IAAIgB,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAEhC;;wBAAMC,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAE/B,OAAO;wBAClC,OAAO,CAACuC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,uBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAW2B,MAAM,GAAGrD,yBAAyB;wBAC/C,MAAM,IAAI+D,oBAAU,CAClB,gBACA,AAAC,0BAAwG/D,OAA/E0B,WAAW2B,MAAM,EAAC,+DAAqF,OAAxBrD,yBAAwB;oBAErI;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE2B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMrE,yBAAyBmD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sHAAsH;oBAChHpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAE/B,OAAO;uBAAG8B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAC9EsB,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAE7D,OAAO;;wBAAGqB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAInC,IAAIiC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAE7D,OAAO;uBAAEqC,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EACnBhE,MACA;;oCA2DqEiE,iBAK7BA,yBA/DjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAGHkD,MAqBFC,OACAC,MACS,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,WAIfC,gBAOS,4BAAA,oBAAA,iBAAA,YAAA,QAAMC,UAUfC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM9E,KAAwB,YAAA,QAAM+E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMV,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CA/DN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAK4E,IAAI,CAAC,AAAC,sCAAqD,OAAhBnB,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAAClC;uDAC3BuB,gBAAgBW,GAAG,CAAC,SAACb;wDAOZrB;oDANP,IAAIqB,QAAQ,QAAQ,OAAOrB,IAAII,OAAO;oDACtC,IAAIiB,QAAQ,UAAU,OAAOrB,IAAI8C,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOrB,IAAIkF,OAAO;oDACxC,IAAI7D,QAAQ,SAAS,OAAOrB,IAAIgD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOrB,IAAImF,UAAU;oDACjD,QAAOnF,gBAAAA,IAAIoF,IAAI,CAAC/D,IAAI,cAAbrB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMK,KAAKgF,QAAQ,CAAC7D,WAAW+C;;;4CAA/B;;;4CAGF,2FAA2F;4CAC3F,4FAA4F;4CAC5F,yFAAyF;4CACzF,2FAA2F;4CAC3F,2FAA2F;4CAC3F,sFAAsF;4CACtF,yFAAyF;4CACnFC,QAAQ9C,MAAMsC,QAAQ,CAACf,MAAM,GAAGvB,MAAMb,QAAQ,CAACoC,MAAM;4CACrDwB,OAAO/C,MAAMlB,KAAK,CAACyC,MAAM,KAAK,KAAKuB,QAAQ3E;4CAClC,mCAAA,4BAAA;iDAAX4E,MAAAA;;;;;;;;;;;;4CAAW,aAAca,yBAAiB;;;mDAA/B,8BAAA,SAAA;;;;4CAAMZ,OAAN;4CAAiC;;gDAAMrE,KAAK4E,IAAI,CAAC,AAAC,wBAA4B,OAALP;;;4CAAxC;;;4CAAjC;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAEf,0FAA0F;4CAC1F,8DAA8D;4CACxDC,iBAAiB,AAAC,qBAAG9D,iBAAU,qBAAGe;iDACpC+C,CAAAA,eAAe1B,MAAM,GAAG,CAAA,GAAxB0B;;;;4CACF;;gDAAMtE,KAAKgF,QAAQ,CACjB,wCACAV,eAAezC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CAAuB;;gDAAMb,KAAKgF,QAAQ,CAACvF,oBAAoBoB,WAAWgB,GAAG,CAACnC;;;4CAAvD;;;4CACZ,mCAAA,4BAAA;iDAAX0E,MAAAA;;;;;;;;;;;;4CAAW,aAAac,uBAAe;;;mDAA5B,8BAAA,SAAA;;;;4CAAMX,MAAN;4CAA8B;;gDAAMvE,KAAK4E,IAAI,CAACL;;;4CAAhB;;;4CAA9B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEX/D,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAKgF,QAAQ,CACjB,4CACAxE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DW,gBAAgB,AAAC,qBAAGhE,iBAAU,qBAAGe;iDACnCiD,CAAAA,cAAc5B,MAAM,GAAG,CAAA,GAAvB4B;;;;4CACF;;gDAAMxE,KAAKgF,QAAQ,CACjB,6CACAR,cAAc3C,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIY;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAa5D,iCAAb,8BAAA,SAAA,0BAAA;oDAAMlB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIwF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMT,aAAN;4DAAiCD,WAAWW,IAAI;gEAAEzF,IAAII,OAAO;gEAAE2E;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAW7B,MAAM,GAAG,CAAA,GAApB6B;;;;4CAAuB;;gDAAMzE,KAAKgF,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAGnE,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BoD,CAAAA,aAAa/B,MAAM,GAAG,CAAA,GAAtB+B;;;;;;;;;;;;4CAA8B,aAAiBjE;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQoB,MAAM,cAAdpB,sCAAAA,qBAAAA,SAAiBjE,MAAM2E,cAActD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAqB;;;;4DADGrB,UAAN;4DACGqB,iBAAiCzE,WAAWgB,GAAG,CAAC,SAAClC;uEAAS;oEAAEwC,MAAMxC,IAAII,OAAO;oEAAEwF,WAAW5F,IAAI4F,SAAS,CAACtB,QAAQI,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMJ,iBAAAA,QAAQuB,KAAK,cAAbvB,qCAAAA,oBAAAA,SAAgBjE,MAAMsF,gBAAgBjE;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQwB,cAAc,cAAtBxB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;2BACAqE,0BAAW;;;oBApEb;oBAuEA,8FAA8F;oBAC9F,qDAAqD;oBAC/CjE,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAMmE,IAAAA,iBAAO,EAAC3F,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,0EAA0E;oBACpEC,UAAUD,YAAY,OAAO,CAAC,IAAIkE,OAAOlE;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAMoE,IAAAA,iBAAO,EAAC7F,MAAM,oBAAoB8F,OAAOrE;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C"}

package/dist/esm/store/sqlite/connection.js

@@ -1,4 +1,4 @@
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 // Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real
 // work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.
 let SqliteStatement = class SqliteStatement {
@@ -39,7 +39,7 @@ export function createConnection(db) {
             await withTransaction(conn, async ()=>{
                 const stmt = db.prepare(sql);
                 for (const row of paramRows)stmt.run(...row);
-            });
+            }, BEGIN_WRITE);
         }
     };
     return conn;

package/dist/esm/store/sqlite/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const stmt = db.prepare(sql);\n        for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAGpD,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBAAgBa,MAAM;gBAC1B,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C;QACF;IACF;IACA,OAAOP;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/connection.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// Wraps node:sqlite's synchronous DatabaseSync in the async Connection/Statement shape every store presents. Every method does its real\n// work synchronously and returns an already-resolved promise, so reconcile's loop, the feature hooks, and runBatch never cross an async boundary internally.\nclass SqliteStatement implements Statement {\n  private stmt: StatementSync;\n\n  constructor(stmt: StatementSync) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...(params as Parameters<StatementSync['run']>));\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...(params as Parameters<StatementSync['get']>));\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...(params as Parameters<StatementSync['all']>));\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...(params as Parameters<StatementSync['iterate']>));\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.setReadBigInts(enabled);\n  }\n}\n\nexport function createConnection(db: DatabaseSync): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new SqliteStatement(db.prepare(sql));\n    },\n    // Prepares once and loops synchronously, wrapped as one async call so features share one\n    // Connection contract with DuckDB. Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so a standalone batch stays atomic and commits once, not per row.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          const stmt = db.prepare(sql);\n          for (const row of paramRows) stmt.run(...(row as Parameters<StatementSync['run']>));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","SqliteStatement","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","row"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAGjE,wIAAwI;AACxI,6JAA6J;AAC7J,IAAA,AAAMC,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAKC;IAC3B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAKF;IAC3B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAKH;IAC3B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAKJ;IAC/B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACK,cAAc,CAACC;IAC3B;IA1BA,YAAYN,IAAmB,CAAE;QAC/B,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASO,iBAAiBC,EAAgB;IAC/C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpBH,GAAGE,IAAI,CAACC;QACV;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAId,gBAAgBW,GAAGI,OAAO,CAACD;QACxC;QACA,yFAAyF;QACzF,4LAA4L;QAC5L,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMnB,gBACJa,MACA;gBACE,MAAMT,OAAOQ,GAAGI,OAAO,CAACD;gBACxB,KAAK,MAAMK,OAAOF,UAAWd,KAAKF,GAAG,IAAKkB;YAC5C,GACArB;QAEJ;IACF;IACA,OAAOc;AACT"}

package/dist/esm/store/sqlite/open.js

@@ -9,7 +9,7 @@ import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { clearCache } from '../cache.js';
 import { getMeta, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 import { createConnection } from './connection.js';
 import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
@@ -64,6 +64,20 @@ async function ensureSchema(conn, cfg, tokenize) {
     if (await getMeta(conn, 'schema_version') === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);
     if (await getMeta(conn, 'features') === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));
 }
+// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY
+// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.
+function setJournalWal(db) {
+    const deadline = Date.now() + 5000;
+    for(;;){
+        try {
+            db.exec('PRAGMA journal_mode = WAL');
+            return;
+        } catch (err) {
+            if (Date.now() >= deadline || !/database is locked|busy/i.test(err.message)) throw err;
+            Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);
+        }
+    }
+}
 async function connect(cfg) {
     const stateDir = join(cfg.baseDir, STATE_DIR);
     mkdirSync(stateDir, {
@@ -76,10 +90,12 @@ async function connect(cfg) {
     let closed = false;
     try {
         var _ref;
-        db.exec('PRAGMA journal_mode = WAL');
-        // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query
-        // that outwaits it still fails loudly.
+        // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive
+        // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.
         db.exec('PRAGMA busy_timeout = 30000');
+        // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy
+        // handler for it, so a concurrent cold open needs its own bounded wait.
+        setJournalWal(db);
         registerFunctions(db, contentTokenize(cfg) === undefined);
         const conn = createConnection(db);
         db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');
@@ -112,7 +128,7 @@ async function connect(cfg) {
                 await withTransaction(conn, async ()=>{
                     await conn.exec('DROP TABLE IF EXISTS content');
                     await createContentTable(conn, tokenize);
-                });
+                }, BEGIN_WRITE);
                 tokenizeOnlyRebuild = true;
             } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
                 // First sight of a resolved weight identity: the model itself hasn't changed, so
@@ -138,7 +154,9 @@ async function connect(cfg) {
             clearCache(cfg);
             return connect(cfg);
         }
-        await ensureSchema(conn, cfg, tokenize);
+        // One writer at a time: the feature hooks check a column then add it, so two cold opens racing
+        // here both see it missing and the second ALTER fails with a duplicate column.
+        await withTransaction(conn, ()=>ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);
         let rebuildWarnings = [];
         if (tokenizeOnlyRebuild) {
             rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);

package/dist/esm/store/sqlite/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    db.exec('PRAGMA journal_mode = WAL');\n    // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n    // that outwaits it still fails loudly.\n    db.exec('PRAGMA busy_timeout = 30000');\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(conn, async () => {\n          await conn.exec('DROP TABLE IF EXISTS content');\n          await createContentTable(conn, tokenize);\n        });\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg, tokenize);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","rekeyChunkText","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","withTransaction","createConnection","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAazB,gBAAgBwB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI1B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEqB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWtC,eAAemB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMhC,QAAQgC,MAAM,sBAAuB,MAAM,MAAM/B,QAAQ+B,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMZ,QAAQgC,MAAM,gBAAiB,MAAM,MAAM/B,QAAQ+B,MAAM,YAAYvC,iBAAiBuB,KAAKlB;AACxG;AAEA,eAAeuC,QAAQrB,GAAmB;IACxC,MAAMsB,WAAWhD,KAAK0B,IAAIuB,OAAO,EAAE7C;IACnCL,UAAUiD,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAASnD,KAAKgD,UAAU3B;IAE9B,MAAMI,KAAK,IAAIxB,aAAakD;IAC5B,6FAA6F;IAC7F,wFAAwF;IACxF,IAAIC,SAAS;IACb,IAAI;YA8E4B;QA7E9B3B,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,uCAAuC;QACvCN,GAAGM,IAAI,CAAC;QACRZ,kBAAkBM,IAAIvB,gBAAgBwB,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAM2B,UAAU,MAAM3C,QAAQgC,MAAM;QACpC,MAAMY,WAAW,MAAM5C,QAAQgC,MAAM;QACrC,MAAMa,eAAepD,iBAAiBuB,KAAKlB;QAC3C,IAAIgD,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY/B,kBAAoBgC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAY/B,gBAAgB;gBAClDmC,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACT3B,GAAGkC,KAAK;gBACRlD,WAAWiB;gBACX,OAAOqB,QAAQrB;YACjB;YACA,MAAMkC,cAAc9C,qBAAqBwC,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,6FAA6F;YAC7F,2FAA2F;YAC3F,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDL,QAAQC,KAAK,CAAC;gBACd,2FAA2F;gBAC3F,yFAAyF;gBACzF,MAAM9C,gBAAgB8B,MAAM;oBAC1B,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC;gBACAa,sBAAsB;YACxB,OAAO,IAAII,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAY/C,qBAAqBuC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAM/C,QAAQ+B,MAAM,YAAYa;YAClC,OAAO;gBACL,MAAMQ,UAAU7C,cAAcoC,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEK,QAAQ,oBAAoB,CAAC;gBACpEX,SAAS;gBACT3B,GAAGkC,KAAK;gBACRlD,WAAWiB;gBACX,OAAOqB,QAAQrB;YACjB;QACF;QAEA,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAMsC,SAAS9B,eAAeT;QAC9B,IAAIuC,WAAW,QAAQA,WAAWrB,UAAU;YAC1Cc,QAAQC,KAAK,CAAC;YACdN,SAAS;YACT3B,GAAGkC,KAAK;YACRlD,WAAWiB;YACX,OAAOqB,QAAQrB;QACjB;QAEA,MAAMkB,aAAaF,MAAMhB,KAAKiB;QAE9B,IAAIsB,kBAA4B,EAAE;QAClC,IAAIT,qBAAqB;YACvBS,kBAAkB,MAAMjD,oBAAoB0B,MAAMhB,KAAKA,IAAIuB,OAAO;YAClE,MAAMtC,QAAQ+B,MAAM,YAAYa;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMW,gBAAgBC,QAAQ,OAAA,MAAMzD,QAAQgC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMvD,UAAUyB,MAAMhB,KAAKA,IAAIuB,OAAO;QAEnE,OAAO;YAAExB;YAAIiB;YAAMhB;YAAKyB;YAAQoB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOxC,KAAK;QACZ,IAAI,CAACoB,QAAQ3B,GAAGkC,KAAK;QACrB,MAAM3B;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAeyC,WAAW/C,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKgD,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQrB;IAC/E,MAAMiD,QAAQvD,YAAYK,IAAIiB,MAAMgC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1F5C,eAAeqC,MAAMiC;IACrB,OAAO;QAAEA,OAAOA;QAAOjD,KAAKgD;QAAavB;QAAQoB;QAAQC;IAAS;AACpE;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMvC,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMyC,KAAKxC,GAAG,EAAC,EAAqByC,CAAC;AAChD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '18';\n\ninterface ConnectResult {\n  db: DatabaseSync;\n  conn: Connection;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// unicode61 splits on spaces, so a language written without them indexes a whole run as one\n// token and word search finds nothing; `content.tokenize` is how such a tree picks trigram.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a DDL string literal where nothing can bind, so the configured\n// value is concatenated; probing a throwaway table first is both the safety and the validation.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// The `_seg` sidecars are appended after path, never inserted: bm25() and snippet(content, 2)\n// are documented against the first three columns. Each holds its field's exploded unspaced runs.\nasync function createContentTable(conn: Connection, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nasync function ensureSchema(conn: Connection, cfg: Config, tokenize: string): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  await createContentTable(conn, tokenize);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) await feature.schema(conn);\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Two processes opening the same fresh tree both try to convert it, and the loser gets SQLITE_BUSY\n// with no busy handler behind it. Bounded because a lock held past this is a real problem, not a race.\nfunction setJournalWal(db: DatabaseSync): void {\n  const deadline = Date.now() + 5000;\n  for (;;) {\n    try {\n      db.exec('PRAGMA journal_mode = WAL');\n      return;\n    } catch (err) {\n      if (Date.now() >= deadline || !/database is locked|busy/i.test((err as Error).message)) throw err;\n      Atomics.wait(new Int32Array(new SharedArrayBuffer(4)), 0, 0, 20);\n    }\n  }\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<ConnectResult> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  // A throw below must release this handle, or on Windows the leaked WAL db is undeletable and\n  // scratch cleanup fails with EPERM/EBUSY. `closed` keeps the catch from double-closing.\n  let closed = false;\n  try {\n    // Before journal_mode, not after: converting a fresh database to WAL takes a brief exclusive\n    // lock, and with no timeout set yet a second process opening the same tree fails in 1ms.\n    db.exec('PRAGMA busy_timeout = 30000');\n    // busy_timeout does not cover the WAL conversion itself: SQLite does not invoke the busy\n    // handler for it, so a concurrent cold open needs its own bounded wait.\n    setJournalWal(db);\n    registerFunctions(db, contentTokenize(cfg) === undefined);\n    const conn = createConnection(db);\n\n    db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n    // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n    const tokenize = resolveTokenize(db, cfg);\n\n    // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n    // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    let tokenizeOnlyRebuild = false;\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n      // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n      if (version !== null && version !== SCHEMA_VERSION) {\n        console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n      const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n      // Only the tokenizer moved, and frontmatter, links, sections and embeddings are file-derived\n      // and tokenizer-independent. Any other signature change takes the full clear/reopen below.\n      if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n        console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n        // One transaction, so a crash before COMMIT rolls back to the old tokenizer's table rather\n        // than to no table. IF EXISTS makes a retry after such a crash a no-op, not a raw error.\n        await withTransaction(\n          conn,\n          async () => {\n            await conn.exec('DROP TABLE IF EXISTS content');\n            await createContentTable(conn, tokenize);\n          },\n          BEGIN_WRITE\n        );\n        tokenizeOnlyRebuild = true;\n      } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n        // First sight of a resolved weight identity: the model itself hasn't changed, so\n        // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n        console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n        await setMeta(conn, 'features', wantFeatures);\n      } else {\n        const changed = signatureDiff(features ?? '', wantFeatures);\n        console.error(`sense: config change (${changed}) rebuilds the index`);\n        closed = true;\n        db.close();\n        clearCache(cfg);\n        return connect(cfg);\n      }\n    }\n\n    // Meta can lie after a crash between table creation and the signature write; the table's own\n    // DDL cannot, so a mismatch here rebuilds whatever meta says.\n    const stored = storedTokenize(db);\n    if (stored !== null && stored !== tokenize) {\n      console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n      closed = true;\n      db.close();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    // One writer at a time: the feature hooks check a column then add it, so two cold opens racing\n    // here both see it missing and the second ALTER fails with a duplicate column.\n    await withTransaction(conn, () => ensureSchema(conn, cfg, tokenize), BEGIN_WRITE);\n\n    let rebuildWarnings: string[] = [];\n    if (tokenizeOnlyRebuild) {\n      rebuildWarnings = await rebuildContentTable(conn, cfg, cfg.baseDir);\n      await setMeta(conn, 'features', wantFeatures);\n    }\n\n    // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n    // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n    const recordedMaxMs = Number((await getMeta(conn, 'reconcile_max_ms')) ?? '0');\n    db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n\n    return { db, conn, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n  } catch (err) {\n    if (!closed) db.close();\n    throw err;\n  }\n}\n\n// The sqlite store's open: connects synchronously (see connect() above), then wraps the\n// resulting connection in the async Store interface.\nexport async function openSqlite(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { db, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(db, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n\nexport async function docCount(store: Store): Promise<number> {\n  const stmt = await store.prepare('SELECT COUNT(*) AS n FROM frontmatter');\n  return ((await stmt.get()) as { n: number }).n;\n}\n"],"names":["mkdirSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","rekeyChunkText","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","BEGIN_WRITE","withTransaction","createConnection","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","createContentTable","conn","tokenize","ensureSchema","feature","schema","setJournalWal","deadline","Date","now","test","Atomics","wait","Int32Array","SharedArrayBuffer","connect","stateDir","baseDir","recursive","dbPath","closed","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","openSqlite","resolvedCfg","store","docCount","stmt","n"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACrF,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAChD,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAEjE,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAmBnC,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMC,mBAAmB;AAEzB,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAa1B,gBAAgByB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI3B,WAAW,kBAAkB,CAAC,kBAAkB,EAAEsB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,8FAA8F;AAC9F,iGAAiG;AACjG,eAAeG,mBAAmBC,IAAgB,EAAEC,QAAgB;IAClE,MAAMD,KAAKX,IAAI,CAAC,CAAC,0IAA0I,EAAEY,SAAS,EAAE,CAAC;AAC3K;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,eAAeC,aAAaF,IAAgB,EAAEhB,GAAW,EAAEiB,QAAgB;IACzE,MAAMD,KAAKX,IAAI,CAAC,CAAC,oIAAoI,CAAC;IACtJ,wFAAwF;IACxF,4FAA4F;IAC5F,MAAMU,mBAAmBC,MAAMC;IAC/B,6FAA6F;IAC7F,qFAAqF;IACrF,MAAMD,KAAKX,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMW,KAAKX,IAAI,CAAC;IAChB,KAAK,MAAMc,WAAWvC,eAAeoB,KAAM,MAAMmB,QAAQC,MAAM,CAACJ;IAChE,IAAI,AAAC,MAAMjC,QAAQiC,MAAM,sBAAuB,MAAM,MAAMhC,QAAQgC,MAAM,kBAAkBpB;IAC5F,IAAI,AAAC,MAAMb,QAAQiC,MAAM,gBAAiB,MAAM,MAAMhC,QAAQgC,MAAM,YAAYxC,iBAAiBwB,KAAKnB;AACxG;AAEA,mGAAmG;AACnG,uGAAuG;AACvG,SAASwC,cAActB,EAAgB;IACrC,MAAMuB,WAAWC,KAAKC,GAAG,KAAK;IAC9B,OAAS;QACP,IAAI;YACFzB,GAAGM,IAAI,CAAC;YACR;QACF,EAAE,OAAOC,KAAK;YACZ,IAAIiB,KAAKC,GAAG,MAAMF,YAAY,CAAC,2BAA2BG,IAAI,CAAC,AAACnB,IAAcC,OAAO,GAAG,MAAMD;YAC9FoB,QAAQC,IAAI,CAAC,IAAIC,WAAW,IAAIC,kBAAkB,KAAK,GAAG,GAAG;QAC/D;IACF;AACF;AAEA,eAAeC,QAAQ9B,GAAmB;IACxC,MAAM+B,WAAW1D,KAAK2B,IAAIgC,OAAO,EAAEvD;IACnCL,UAAU2D,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS7D,KAAK0D,UAAUpC;IAE9B,MAAMI,KAAK,IAAIzB,aAAa4D;IAC5B,6FAA6F;IAC7F,wFAAwF;IACxF,IAAIC,SAAS;IACb,IAAI;YAsF4B;QArF9B,6FAA6F;QAC7F,yFAAyF;QACzFpC,GAAGM,IAAI,CAAC;QACR,yFAAyF;QACzF,wEAAwE;QACxEgB,cAActB;QACdN,kBAAkBM,IAAIxB,gBAAgByB,SAASE;QAC/C,MAAMc,OAAO7B,iBAAiBY;QAE9BA,GAAGM,IAAI,CAAC;QAER,2FAA2F;QAC3F,8FAA8F;QAC9F,MAAMY,WAAWnB,gBAAgBC,IAAIC;QAErC,uFAAuF;QACvF,qGAAqG;QACrG,MAAMoC,UAAU,MAAMrD,QAAQiC,MAAM;QACpC,MAAMqB,WAAW,MAAMtD,QAAQiC,MAAM;QACrC,MAAMsB,eAAe9D,iBAAiBwB,KAAKnB;QAC3C,IAAI0D,sBAAsB;QAC1B,IAAI,AAACH,YAAY,QAAQA,YAAYxC,kBAAoByC,aAAa,QAAQA,aAAaC,cAAe;YACxG,uFAAuF;YACvF,wFAAwF;YACxF,IAAIF,YAAY,QAAQA,YAAYxC,gBAAgB;gBAClD4C,QAAQC,KAAK,CAAC;gBACdN,SAAS;gBACTpC,GAAG2C,KAAK;gBACR5D,WAAWkB;gBACX,OAAO8B,QAAQ9B;YACjB;YACA,MAAM2C,cAAcvD,qBAAqBiD,qBAAAA,sBAAAA,WAAY,IAAIC;YACzD,6FAA6F;YAC7F,2FAA2F;YAC3F,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;gBACzDL,QAAQC,KAAK,CAAC;gBACd,2FAA2F;gBAC3F,yFAAyF;gBACzF,MAAMvD,gBACJ8B,MACA;oBACE,MAAMA,KAAKX,IAAI,CAAC;oBAChB,MAAMU,mBAAmBC,MAAMC;gBACjC,GACAhC;gBAEFsD,sBAAsB;YACxB,OAAO,IAAII,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAYxD,qBAAqBgD,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;gBACnH,iFAAiF;gBACjF,wFAAwF;gBACxFE,QAAQC,KAAK,CAAC;gBACd,MAAMzD,QAAQgC,MAAM,YAAYsB;YAClC,OAAO;gBACL,MAAMQ,UAAUtD,cAAc6C,qBAAAA,sBAAAA,WAAY,IAAIC;gBAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEK,QAAQ,oBAAoB,CAAC;gBACpEX,SAAS;gBACTpC,GAAG2C,KAAK;gBACR5D,WAAWkB;gBACX,OAAO8B,QAAQ9B;YACjB;QACF;QAEA,6FAA6F;QAC7F,8DAA8D;QAC9D,MAAM+C,SAASvC,eAAeT;QAC9B,IAAIgD,WAAW,QAAQA,WAAW9B,UAAU;YAC1CuB,QAAQC,KAAK,CAAC;YACdN,SAAS;YACTpC,GAAG2C,KAAK;YACR5D,WAAWkB;YACX,OAAO8B,QAAQ9B;QACjB;QAEA,+FAA+F;QAC/F,+EAA+E;QAC/E,MAAMd,gBAAgB8B,MAAM,IAAME,aAAaF,MAAMhB,KAAKiB,WAAWhC;QAErE,IAAI+D,kBAA4B,EAAE;QAClC,IAAIT,qBAAqB;YACvBS,kBAAkB,MAAM1D,oBAAoB0B,MAAMhB,KAAKA,IAAIgC,OAAO;YAClE,MAAMhD,QAAQgC,MAAM,YAAYsB;QAClC;QAEA,wFAAwF;QACxF,2FAA2F;QAC3F,MAAMW,gBAAgBC,QAAQ,OAAA,MAAMnE,QAAQiC,MAAM,iCAApB,kBAAA,OAA4C;QAC1EjB,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAE8C,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;QAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMhE,UAAUyB,MAAMhB,KAAKA,IAAIgC,OAAO;QAEnE,OAAO;YAAEjC;YAAIiB;YAAMhB;YAAKkC;YAAQoB;YAAQC,UAAU;mBAAIP;mBAAoBO;aAAS;QAAC;IACtF,EAAE,OAAOjD,KAAK;QACZ,IAAI,CAAC6B,QAAQpC,GAAG2C,KAAK;QACrB,MAAMpC;IACR;AACF;AAEA,wFAAwF;AACxF,qDAAqD;AACrD,OAAO,eAAekD,WAAWxD,GAAmB;IAClD,MAAM,EAAED,EAAE,EAAEiB,IAAI,EAAEhB,KAAKyD,WAAW,EAAEvB,MAAM,EAAEoB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMzB,QAAQ9B;IAC/E,MAAM0D,QAAQhE,YAAYK,IAAIiB,MAAMyC,aAAaA,YAAYzB,OAAO;IACpE,0FAA0F;IAC1FtD,eAAesC,MAAM0C;IACrB,OAAO;QAAEA,OAAOA;QAAO1D,KAAKyD;QAAavB;QAAQoB;QAAQC;IAAS;AACpE;AAEA,OAAO,eAAeI,SAASD,KAAY;IACzC,MAAME,OAAO,MAAMF,MAAMhD,OAAO,CAAC;IACjC,OAAO,AAAE,CAAA,MAAMkD,KAAKjD,GAAG,EAAC,EAAqBkD,CAAC;AAChD"}

package/dist/esm/store/sqlite/reconcile.js

@@ -6,7 +6,7 @@ import { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.js';
 import { reparseFiles } from '../../scan/reparse.js';
 import { segmentField } from '../../text/segment.js';
 import { getColumns, getMeta, quoteIdent, setMeta } from '../shared.js';
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 // Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last
 // computed value on every touch, not just the reconciles that recompute it.
 const CORE_FRONTMATTER_COLUMNS = new Set([
@@ -87,7 +87,10 @@ export async function reconcile(conn, cfg, baseDir) {
     const txStart = Date.now();
     await withTransaction(conn, async ()=>{
         var _feature_remove, _feature_afterReconcile;
-        for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
+        // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a
+        // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.
+        const present = await getColumns(conn);
+        for (const col of newColumns)if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);
         // content's rowid lookup depends on its coupled frontmatter row, so the content
         // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.
         if (vanished.length > 0) {
@@ -151,7 +154,7 @@ export async function reconcile(conn, cfg, baseDir) {
             await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
         }
         for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
-    });
+    }, BEGIN_WRITE);
     // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):
     // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.
     const durationMs = Date.now() - txStart;
@@ -214,6 +217,6 @@ export async function rebuildContentTable(conn, cfg, baseDir) {
     }
     await withTransaction(conn, async ()=>{
         if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);
-    });
+    }, BEGIN_WRITE);
     return warnings;
 }

package/dist/esm/store/sqlite/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    // content's rowid lookup depends on its coupled frontmatter row, so the content\n    // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n    if (vanished.length > 0) {\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        vanished.map((p) => [p])\n      );\n    }\n    if (reparsedExisting.length > 0) {\n      // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n      // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n      await conn.runBatch(\n        'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n        reparsedExisting.map((p) => [p])\n      );\n    }\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      await conn.runBatch(\n        INSERT_CONTENT_SQL,\n        parsedDocs.map((doc) => contentRow(doc, segmenting))\n      );\n    }\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(conn, async () => {\n    if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n  });\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAaZ,aAAaW,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAaZ,aAAaW,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAaZ,aAAaW,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWrD,eAAe0B;IAChC,MAAM4B,cAAc,MAAM/C,WAAWkB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAAStD,SAAS,mBAAmB0C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMrD,aAAasC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIf,WACR,gBACA,CAAC,uBAAuB,EAAEgE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC7D,iBAAiBsC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACrB,YAAY0D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGxD,WAAWwD,GAAG,YAAY,EAAExD,WAAWwD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBAAgBc,MAAM;YA8DyCqD,iBAK7BA;QAlEtC,KAAK,MAAMjB,OAAOH,WAAY,MAAMjC,KAAKsD,IAAI,CAAC,CAAC,mCAAmC,EAAEtE,WAAWoD,MAAM;QAErG,gFAAgF;QAChF,oGAAoG;QACpG,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,6FAA6F;YAC7F,uGAAuG;YACvG,MAAMzB,KAAKuD,QAAQ,CACjB,sFACAP,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM+B,OAAOxB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIiE,OAAO;oBACxC,IAAIrB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAIkE,UAAU;oBACjD,QAAOlE,gBAAAA,IAAImE,IAAI,CAACvB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKuD,QAAQ,CAACd,WAAWe;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAM/D,aAAapB,gBAAgB4B,SAAS2D;YAC5C,MAAM5D,KAAKuD,QAAQ,CACjBjE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKuD,QAAQ,CACjB,4CACAxC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMY,gBAAgB;eAAI9C;eAAaiC;SAAiB;QACxD,IAAIa,cAAcpC,MAAM,GAAG,GACzB,MAAMzB,KAAKuD,QAAQ,CACjB,6CACAM,cAAcxD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMa,aAA0B,EAAE;QAClC,KAAK,MAAMtE,OAAOwC,WAAY,KAAK,MAAM+B,cAAcvE,IAAIwE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACzE,IAAIE,OAAO;YAAEqE;SAAW;QAC7G,IAAID,WAAWrC,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAInD;eAAaiC;SAAiB;QACvD,IAAIkB,aAAazC,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBrD,MAAMkE,cAActB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMe,iBAAiCpC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE2E,WAAW7E,IAAI6E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBrD,MAAMoE,gBAAgBxB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAM6B,aAAatB,KAAKC,GAAG,KAAKF;IAChC,MAAMwB,UAAU,MAAM3F,QAAQiB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM2E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM1F,QAAQe,MAAM,oBAAoB6E,OAAOJ;IAEzE,OAAO;QAAE/C,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASmD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG3C,IAAI,CAAC,OAAOwC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI3E,IAAI2E,IAAIH,KAAK,CAAC,KAAK/E,GAAG,CAAC,CAAC6E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAItG;IACpB,KAAK,MAAM,CAACuG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAEpE,GAAG,CAACuE,SAASC,KAAKF,QAAQrD,GAAG,CAACsD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAExE,GAAG,CAAC0E,MAAMD,QAAQrD,GAAG,CAACsD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAC/C,IAAMA,EAAEkC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQlE,IAAI,KAAK,IAAI,aAAa;WAAIkE;KAAQ,CAACrF,GAAG,CAAC+F,OAAO1D,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAe2D,oBAAoBrG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMoG,OAAO,MAAMtG,KAAKQ,OAAO,CAAC;IAChC,MAAM+F,QAAQ,IAAInH,IAAI,AAAE,CAAA,MAAMkH,KAAK5F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ1B,UAAUwB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMiG,MAAMtF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAapB,gBAAgB4B,SAAS2D;IAC5C,MAAMjC,WAAqB,EAAE;IAC7B,MAAM6B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQrG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU8E,YAAY,EAAE,GAAG/H,UAAU8H;QAClD7E,SAASsC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC1E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBAAgBc,MAAM;QAC1B,IAAIwD,KAAK/B,MAAM,GAAG,GAAG,MAAMzB,KAAKuD,QAAQ,CAACjE,oBAAoBkE;IAC/D;IACA,OAAO7B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/sqlite/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { contentTokenize } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { segmentField } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// A single content row's param tuple, matching INSERT_CONTENT_SQL's placeholder order.\n// Assumes the frontmatter row for doc.relPath already exists (rowid subquery).\nfunction contentRow(doc: ParsedDoc, segmenting: boolean): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : ''];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(\n    conn,\n    async () => {\n      // Re-read inside the write transaction: newColumns came from a read taken before it opened, so a\n      // concurrent reconcile may have added some of them since. ALTER has no IF NOT EXISTS.\n      const present = await getColumns(conn);\n      for (const col of newColumns) if (!present.has(col)) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n      // content's rowid lookup depends on its coupled frontmatter row, so the content\n      // deletes/inserts run while every path still has one -- the vanished frontmatter delete comes last.\n      if (vanished.length > 0) {\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          vanished.map((p) => [p])\n        );\n      }\n      if (reparsedExisting.length > 0) {\n        // FTS5 has no upsert, so delete-before-insert, coupled to the frontmatter rowid (indexed via\n        // its PRIMARY KEY) rather than the UNINDEXED `path` column, which a per-row DELETE would scan to find.\n        await conn.runBatch(\n          'DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)',\n          reparsedExisting.map((p) => [p])\n        );\n      }\n\n      if (parsedDocs.length > 0) {\n        const rows = parsedDocs.map((doc) =>\n          writableColumns.map((col) => {\n            if (col === 'path') return doc.relPath;\n            if (col === '_mtime') return doc.mtimeMs;\n            if (col === '_ctime') return doc.ctimeMs;\n            if (col === '_size') return doc.size;\n            // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n            if (col === '_parse_error') return doc.parseError;\n            return doc.data[col] ?? null;\n          })\n        );\n        await conn.runBatch(insertSql, rows);\n\n        // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n        // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n        const segmenting = contentTokenize(cfg) === undefined;\n        await conn.runBatch(\n          INSERT_CONTENT_SQL,\n          parsedDocs.map((doc) => contentRow(doc, segmenting))\n        );\n      }\n\n      if (vanished.length > 0)\n        await conn.runBatch(\n          'DELETE FROM frontmatter WHERE \"path\" = ?',\n          vanished.map((p) => [p])\n        );\n\n      // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n      // new docs have nothing to clear, which keeps cold builds linear.\n      const presetTouched = [...vanished, ...reparsedExisting];\n      if (presetTouched.length > 0)\n        await conn.runBatch(\n          'DELETE FROM preset_files WHERE \"path\" = ?',\n          presetTouched.map((p) => [p])\n        );\n      const presetRows: unknown[][] = [];\n      for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n      if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n      const removedPaths = [...vanished, ...reparsedExisting];\n      if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n      for (const feature of features) {\n        const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n        await feature.store?.(conn, docsForFeature, delta);\n      }\n      for (const feature of features) await feature.afterReconcile?.(conn, delta);\n    },\n    BEGIN_WRITE\n  );\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// Tokenize-only rebuild: only `content` is repopulated (the other tables are tokenizer-independent)\n// and no feature extractors run. Returns per-file warnings, since mtimes stay untouched and reconcile would never reparse them.\nexport async function rebuildContentTable(conn: Connection, cfg: Config, baseDir: string): Promise<string[]> {\n  const stmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const known = new Set(((await stmt.all()) as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const warnings: string[] = [];\n  const rows: unknown[][] = [];\n  for (const file of files) {\n    const { doc, warnings: fileWarnings } = parseFile(file);\n    warnings.push(...fileWarnings);\n    rows.push(contentRow(doc, segmenting));\n  }\n  await withTransaction(\n    conn,\n    async () => {\n      if (rows.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, rows);\n    },\n    BEGIN_WRITE\n  );\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","reparseFiles","segmentField","getColumns","getMeta","quoteIdent","setMeta","BEGIN_WRITE","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","segmenting","relPath","search","title","summary","text","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","txStart","Date","now","feature","present","exec","runBatch","rows","ctimeMs","parseError","data","undefined","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","key","val","keys","embedIdentityAdopted","embedPart","find","at","indexOf","signatureDiff","label","rebuildContentTable","stmt","known","file","fileWarnings"],"mappings":"AACA,SAASA,eAAe,QAAQ,wBAAwB;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAC7E,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,eAAe;AACxE,SAASC,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAGjE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,uFAAuF;AACvF,+EAA+E;AAC/E,SAASC,WAAWC,GAAc,EAAEC,UAAmB;IACrD,OAAO;QAACD,IAAIE,OAAO;QAAEF,IAAIG,MAAM,CAACC,KAAK;QAAEJ,IAAIG,MAAM,CAACE,OAAO;QAAEL,IAAIG,MAAM,CAACG,IAAI;QAAEN,IAAIE,OAAO;QAAED,aAAab,aAAaY,IAAIG,MAAM,CAACC,KAAK,IAAI;QAAIH,aAAab,aAAaY,IAAIG,MAAM,CAACE,OAAO,IAAI;QAAIJ,aAAab,aAAaY,IAAIG,MAAM,CAACG,IAAI,IAAI;KAAG;AACjP;AAEA,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQ3B,UAAUyB,KAAKC;IAC7B,MAAME,aAAa,IAAIhB,IAAIe,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEZ,OAAO;QAClC,OAAO,CAACyB,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWtD,eAAe2B;IAChC,MAAM4B,cAAc,MAAMhD,WAAWmB;IAErC,oFAAoF;IACpF,uDAAuD;IACvD,MAAM8B,SAASvD,SAAS,mBAAmB2C,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAG,MAAMtD,aAAauC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IACxHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIS,WAAWb,MAAM,GAAGpC,yBAAyB;QAC/C,MAAM,IAAIhB,WACR,gBACA,CAAC,uBAAuB,EAAEiE,WAAWb,MAAM,CAAC,0EAA0E,EAAEpC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAMkD,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMrD,yBAAyB8B,GAAG,CAACuB,MAAM,CAAC9D,iBAAiBuC,GAAG,CAACuB;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACtB,YAAY2D,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGzD,WAAWyD,GAAG,YAAY,EAAEzD,WAAWyD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAMkD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO;QAAGiD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI3D,IAAIuD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAEpD,OAAO,EAAEsB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAMC,UAAUC,KAAKC,GAAG;IACxB,MAAMlE,gBACJc,MACA;YAiEqEqD,iBAK7BA;QArEtC,iGAAiG;QACjG,sFAAsF;QACtF,MAAMC,UAAU,MAAMzE,WAAWmB;QACjC,KAAK,MAAMoC,OAAOH,WAAY,IAAI,CAACqB,QAAQrC,GAAG,CAACmB,MAAM,MAAMpC,KAAKuD,IAAI,CAAC,CAAC,mCAAmC,EAAExE,WAAWqD,MAAM;QAE5H,gFAAgF;QAChF,oGAAoG;QACpG,IAAIrB,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMzB,KAAKwD,QAAQ,CACjB,sFACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAE3B;QACA,IAAID,iBAAiBvB,MAAM,GAAG,GAAG;YAC/B,6FAA6F;YAC7F,uGAAuG;YACvG,MAAMzB,KAAKwD,QAAQ,CACjB,sFACAR,iBAAiB3C,GAAG,CAAC,CAAC4C,IAAM;oBAACA;iBAAE;QAEnC;QAEA,IAAIjB,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAMgC,OAAOzB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAOZ5C;oBANP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIE,OAAO;oBACtC,IAAI0C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAIkE,OAAO;oBACxC,IAAItB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAImE,UAAU;oBACjD,QAAOnE,gBAAAA,IAAIoE,IAAI,CAACxB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWgB;YAE/B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMhE,aAAarB,gBAAgB6B,SAAS4D;YAC5C,MAAM7D,KAAKwD,QAAQ,CACjBlE,oBACA0C,WAAW3B,GAAG,CAAC,CAACb,MAAQD,WAAWC,KAAKC;QAE5C;QAEA,IAAIsB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMa,gBAAgB;eAAI/C;eAAaiC;SAAiB;QACxD,IAAIc,cAAcrC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAM,cAAczD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMc,aAA0B,EAAE;QAClC,KAAK,MAAMvE,OAAOwC,WAAY,KAAK,MAAMgC,cAAcxE,IAAIyE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAAC1E,IAAIE,OAAO;YAAEsE;SAAW;QAC7G,IAAID,WAAWtC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DO;QAE1G,MAAMI,eAAe;eAAIpD;eAAaiC;SAAiB;QACvD,IAAImB,aAAa1C,MAAM,GAAG,GAAG,KAAK,MAAM4B,WAAWzB,SAAU,QAAMyB,kBAAAA,QAAQe,MAAM,cAAdf,sCAAAA,qBAAAA,SAAiBrD,MAAMmE,cAAcvB;QACxG,KAAK,MAAMS,WAAWzB,SAAU;gBAExByB;YADN,MAAMgB,iBAAiCrC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIE,OAAO;oBAAE4E,WAAW9E,IAAI8E,SAAS,CAACjB,QAAQkB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMlB,iBAAAA,QAAQmB,KAAK,cAAbnB,qCAAAA,oBAAAA,SAAgBrD,MAAMqE,gBAAgBzB;QAC9C;QACA,KAAK,MAAMS,WAAWzB,SAAU,QAAMyB,0BAAAA,QAAQoB,cAAc,cAAtBpB,8CAAAA,6BAAAA,SAAyBrD,MAAM4C;IACvE,GACA3D;IAGF,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMyF,aAAavB,KAAKC,GAAG,KAAKF;IAChC,MAAMyB,UAAU,MAAM7F,QAAQkB,MAAM;IACpC,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM4E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS,MAAM5F,QAAQgB,MAAM,oBAAoB8E,OAAOJ;IAEzE,OAAO;QAAEhD,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASoD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAG5C,IAAI,CAAC,OAAOyC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAI5E,IAAI4E,IAAIH,KAAK,CAAC,KAAKhF,GAAG,CAAC,CAAC8E,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAIvG;IACpB,KAAK,MAAM,CAACwG,KAAKC,IAAI,IAAIH,EAAG,IAAID,EAAErE,GAAG,CAACwE,SAASC,KAAKF,QAAQtD,GAAG,CAACuD;IAChE,KAAK,MAAMA,OAAOH,EAAEK,IAAI,GAAI,IAAI,CAACJ,EAAEzE,GAAG,CAAC2E,MAAMD,QAAQtD,GAAG,CAACuD;IACzD,OAAOD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASI,qBAAqBf,MAAc,EAAEC,KAAa;IAChE,MAAMe,YAAY,CAACR,MAAgBA,IAAIH,KAAK,CAAC,KAAKY,IAAI,CAAC,CAAChD,IAAMA,EAAEmC,UAAU,CAAC;IAC3E,MAAMM,IAAIM,UAAUhB;IACpB,MAAMS,IAAIO,UAAUf;IACpB,IAAIS,MAAM7B,aAAa4B,MAAM5B,WAAW,OAAO;IAC/C,MAAMqC,KAAKT,EAAEU,OAAO,CAAC;IACrB,OAAOT,EAAES,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKT,EAAEH,KAAK,CAAC,GAAGY,QAAQR;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASU,cAAcpB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMoB,QAAQ,CAACT,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAIR,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAEQ,IAAIN,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQnE,IAAI,KAAK,IAAI,aAAa;WAAImE;KAAQ,CAACtF,GAAG,CAACgG,OAAO3D,IAAI,CAAC;AACxE;AAEA,oGAAoG;AACpG,gIAAgI;AAChI,OAAO,eAAe4D,oBAAoBtG,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,MAAMqG,OAAO,MAAMvG,KAAKQ,OAAO,CAAC;IAChC,MAAMgG,QAAQ,IAAIpH,IAAI,AAAE,CAAA,MAAMmH,KAAK7F,GAAG,EAAC,EAA+BL,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IACvF,MAAMX,QAAQ3B,UAAUyB,KAAKC,SAASc,MAAM,CAAC,CAACV,IAAMkG,MAAMvF,GAAG,CAACX,EAAEZ,OAAO;IACvE,MAAMD,aAAarB,gBAAgB6B,SAAS4D;IAC5C,MAAMlC,WAAqB,EAAE;IAC7B,MAAM8B,OAAoB,EAAE;IAC5B,KAAK,MAAMgD,QAAQtG,MAAO;QACxB,MAAM,EAAEX,GAAG,EAAEmC,UAAU+E,YAAY,EAAE,GAAGjI,UAAUgI;QAClD9E,SAASuC,IAAI,IAAIwC;QACjBjD,KAAKS,IAAI,CAAC3E,WAAWC,KAAKC;IAC5B;IACA,MAAMP,gBACJc,MACA;QACE,IAAIyD,KAAKhC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoBmE;IAC/D,GACAxE;IAEF,OAAO0C;AACT"}

package/dist/esm/store/transaction.d.ts

@@ -1,5 +1,6 @@
 interface ExecConnection {
     exec(sql: string): Promise<void>;
 }
-export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T>;
+export declare const BEGIN_WRITE = "BEGIN IMMEDIATE";
+export declare function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin?: string): Promise<T>;
 export {};

package/dist/esm/store/transaction.js

@@ -14,9 +14,9 @@ function stateFor(conn) {
     }
     return state;
 }
-async function enter(conn, state) {
+async function enter(conn, state, begin) {
     if (state.depth === 0) {
-        await conn.exec('BEGIN');
+        await conn.exec(begin);
         state.rollbackOnly = false;
     }
     state.depth++;
@@ -37,11 +37,16 @@ async function leaveErr(conn, state) {
     state.depth--;
     if (state.depth === 0) await conn.exec('ROLLBACK');
 }
-// Depth-0 opens BEGIN; a nested call joins it and issues no SQL of its own. The outermost call
-// owns COMMIT/ROLLBACK; every store and internal write loop goes through this one helper, so join semantics never differ by call site.
-export async function withTransaction(conn, fn) {
+// A writer takes its lock at BEGIN rather than on first write: SQLite will not retry a deferred
+// transaction that read first and then upgrades, so busy_timeout never applies and the second
+// concurrent command fails outright instead of queueing. Reads keep plain BEGIN, or every snapshot
+// read (map, peek) would serialize against writers. DuckDB has no IMMEDIATE and passes nothing.
+export const BEGIN_WRITE = 'BEGIN IMMEDIATE';
+// Depth-0 opens the transaction; a nested call joins it and issues no SQL of its own. The outermost
+// call owns COMMIT/ROLLBACK, so join semantics never differ by call site.
+export async function withTransaction(conn, fn, begin = 'BEGIN') {
     const state = stateFor(conn);
-    await enter(conn, state);
+    await enter(conn, state, begin);
     let result;
     try {
         result = await fn();

package/dist/esm/store/transaction.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n  exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n  depth: number;\n  rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n  let state = states.get(conn);\n  if (!state) {\n    state = { depth: 0, rollbackOnly: false };\n    states.set(conn, state);\n  }\n  return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState): Promise<void> {\n  if (state.depth === 0) {\n    await conn.exec('BEGIN');\n    state.rollbackOnly = false;\n  }\n  state.depth++;\n}\n\n// Reaching depth 0 still rollback-only means an inner scope failed and an enclosing catch\n// swallowed it; commit here would persist a partial write, so this rolls back and throws instead.\nasync function leaveOk(conn: ExecConnection, state: TxState): Promise<void> {\n  state.depth--;\n  if (state.depth > 0) return;\n  if (state.rollbackOnly) {\n    await conn.exec('ROLLBACK');\n    throw new Error('transaction rolled back: a nested scope failed');\n  }\n  await conn.exec('COMMIT');\n}\n\nasync function leaveErr(conn: ExecConnection, state: TxState): Promise<void> {\n  state.rollbackOnly = true;\n  state.depth--;\n  if (state.depth === 0) await conn.exec('ROLLBACK');\n}\n\n// Depth-0 opens BEGIN; a nested call joins it and issues no SQL of its own. The outermost call\n// owns COMMIT/ROLLBACK; every store and internal write loop goes through this one helper, so join semantics never differ by call site.\nexport async function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>): Promise<T> {\n  const state = stateFor(conn);\n  await enter(conn, state);\n  let result: T;\n  try {\n    result = await fn();\n  } catch (err) {\n    await leaveErr(conn, state);\n    throw err;\n  }\n  await leaveOk(conn, state);\n  return result;\n}\n"],"names":["states","WeakMap","stateFor","conn","state","get","depth","rollbackOnly","set","enter","exec","leaveOk","Error","leaveErr","withTransaction","fn","result","err"],"mappings":"AAAA,sFAAsF;AACtF,uHAAuH;AAUvH,0FAA0F;AAC1F,kEAAkE;AAClE,MAAMA,SAAS,IAAIC;AAEnB,SAASC,SAASC,IAAoB;IACpC,IAAIC,QAAQJ,OAAOK,GAAG,CAACF;IACvB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;YAAGC,cAAc;QAAM;QACxCP,OAAOQ,GAAG,CAACL,MAAMC;IACnB;IACA,OAAOA;AACT;AAEA,eAAeK,MAAMN,IAAoB,EAAEC,KAAc;IACvD,IAAIA,MAAME,KAAK,KAAK,GAAG;QACrB,MAAMH,KAAKO,IAAI,CAAC;QAChBN,MAAMG,YAAY,GAAG;IACvB;IACAH,MAAME,KAAK;AACb;AAEA,0FAA0F;AAC1F,kGAAkG;AAClG,eAAeK,QAAQR,IAAoB,EAAEC,KAAc;IACzDA,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,GAAG,GAAG;IACrB,IAAIF,MAAMG,YAAY,EAAE;QACtB,MAAMJ,KAAKO,IAAI,CAAC;QAChB,MAAM,IAAIE,MAAM;IAClB;IACA,MAAMT,KAAKO,IAAI,CAAC;AAClB;AAEA,eAAeG,SAASV,IAAoB,EAAEC,KAAc;IAC1DA,MAAMG,YAAY,GAAG;IACrBH,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,KAAK,GAAG,MAAMH,KAAKO,IAAI,CAAC;AACzC;AAEA,+FAA+F;AAC/F,uIAAuI;AACvI,OAAO,eAAeI,gBAAmBX,IAAoB,EAAEY,EAAoB;IACjF,MAAMX,QAAQF,SAASC;IACvB,MAAMM,MAAMN,MAAMC;IAClB,IAAIY;IACJ,IAAI;QACFA,SAAS,MAAMD;IACjB,EAAE,OAAOE,KAAK;QACZ,MAAMJ,SAASV,MAAMC;QACrB,MAAMa;IACR;IACA,MAAMN,QAAQR,MAAMC;IACpB,OAAOY;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/transaction.ts"],"sourcesContent":["// Store-agnostic joining transaction helper: node:sqlite and DuckDB both support only\n// BEGIN/COMMIT/ROLLBACK, no SAVEPOINT -- nesting joins the outer transaction, sharing one semantics through exec(sql).\ninterface ExecConnection {\n  exec(sql: string): Promise<void>;\n}\n\ninterface TxState {\n  depth: number;\n  rollbackOnly: boolean;\n}\n\n// Keyed by connection, not a module singleton: two connections can coexist in one process\n// (e.g. watch.ts alongside a CLI query) and must not share depth.\nconst states = new WeakMap<ExecConnection, TxState>();\n\nfunction stateFor(conn: ExecConnection): TxState {\n  let state = states.get(conn);\n  if (!state) {\n    state = { depth: 0, rollbackOnly: false };\n    states.set(conn, state);\n  }\n  return state;\n}\n\nasync function enter(conn: ExecConnection, state: TxState, begin: string): Promise<void> {\n  if (state.depth === 0) {\n    await conn.exec(begin);\n    state.rollbackOnly = false;\n  }\n  state.depth++;\n}\n\n// Reaching depth 0 still rollback-only means an inner scope failed and an enclosing catch\n// swallowed it; commit here would persist a partial write, so this rolls back and throws instead.\nasync function leaveOk(conn: ExecConnection, state: TxState): Promise<void> {\n  state.depth--;\n  if (state.depth > 0) return;\n  if (state.rollbackOnly) {\n    await conn.exec('ROLLBACK');\n    throw new Error('transaction rolled back: a nested scope failed');\n  }\n  await conn.exec('COMMIT');\n}\n\nasync function leaveErr(conn: ExecConnection, state: TxState): Promise<void> {\n  state.rollbackOnly = true;\n  state.depth--;\n  if (state.depth === 0) await conn.exec('ROLLBACK');\n}\n\n// A writer takes its lock at BEGIN rather than on first write: SQLite will not retry a deferred\n// transaction that read first and then upgrades, so busy_timeout never applies and the second\n// concurrent command fails outright instead of queueing. Reads keep plain BEGIN, or every snapshot\n// read (map, peek) would serialize against writers. DuckDB has no IMMEDIATE and passes nothing.\nexport const BEGIN_WRITE = 'BEGIN IMMEDIATE';\n\n// Depth-0 opens the transaction; a nested call joins it and issues no SQL of its own. The outermost\n// call owns COMMIT/ROLLBACK, so join semantics never differ by call site.\nexport async function withTransaction<T>(conn: ExecConnection, fn: () => Promise<T>, begin = 'BEGIN'): Promise<T> {\n  const state = stateFor(conn);\n  await enter(conn, state, begin);\n  let result: T;\n  try {\n    result = await fn();\n  } catch (err) {\n    await leaveErr(conn, state);\n    throw err;\n  }\n  await leaveOk(conn, state);\n  return result;\n}\n"],"names":["states","WeakMap","stateFor","conn","state","get","depth","rollbackOnly","set","enter","begin","exec","leaveOk","Error","leaveErr","BEGIN_WRITE","withTransaction","fn","result","err"],"mappings":"AAAA,sFAAsF;AACtF,uHAAuH;AAUvH,0FAA0F;AAC1F,kEAAkE;AAClE,MAAMA,SAAS,IAAIC;AAEnB,SAASC,SAASC,IAAoB;IACpC,IAAIC,QAAQJ,OAAOK,GAAG,CAACF;IACvB,IAAI,CAACC,OAAO;QACVA,QAAQ;YAAEE,OAAO;YAAGC,cAAc;QAAM;QACxCP,OAAOQ,GAAG,CAACL,MAAMC;IACnB;IACA,OAAOA;AACT;AAEA,eAAeK,MAAMN,IAAoB,EAAEC,KAAc,EAAEM,KAAa;IACtE,IAAIN,MAAME,KAAK,KAAK,GAAG;QACrB,MAAMH,KAAKQ,IAAI,CAACD;QAChBN,MAAMG,YAAY,GAAG;IACvB;IACAH,MAAME,KAAK;AACb;AAEA,0FAA0F;AAC1F,kGAAkG;AAClG,eAAeM,QAAQT,IAAoB,EAAEC,KAAc;IACzDA,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,GAAG,GAAG;IACrB,IAAIF,MAAMG,YAAY,EAAE;QACtB,MAAMJ,KAAKQ,IAAI,CAAC;QAChB,MAAM,IAAIE,MAAM;IAClB;IACA,MAAMV,KAAKQ,IAAI,CAAC;AAClB;AAEA,eAAeG,SAASX,IAAoB,EAAEC,KAAc;IAC1DA,MAAMG,YAAY,GAAG;IACrBH,MAAME,KAAK;IACX,IAAIF,MAAME,KAAK,KAAK,GAAG,MAAMH,KAAKQ,IAAI,CAAC;AACzC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,mGAAmG;AACnG,gGAAgG;AAChG,OAAO,MAAMI,cAAc,kBAAkB;AAE7C,oGAAoG;AACpG,0EAA0E;AAC1E,OAAO,eAAeC,gBAAmBb,IAAoB,EAAEc,EAAoB,EAAEP,QAAQ,OAAO;IAClG,MAAMN,QAAQF,SAASC;IACvB,MAAMM,MAAMN,MAAMC,OAAOM;IACzB,IAAIQ;IACJ,IAAI;QACFA,SAAS,MAAMD;IACjB,EAAE,OAAOE,KAAK;QACZ,MAAML,SAASX,MAAMC;QACrB,MAAMe;IACR;IACA,MAAMP,QAAQT,MAAMC;IACpB,OAAOc;AACT"}

package/dist/esm/store/turso/connection.js

@@ -1,4 +1,4 @@
-import { withTransaction } from '../transaction.js';
+import { BEGIN_WRITE, withTransaction } from '../transaction.js';
 // Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this
 // wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.
 let TursoStatementWrapper = class TursoStatementWrapper {
@@ -41,7 +41,7 @@ export function createConnection(db) {
                         sql,
                         args
                     })));
-            });
+            }, BEGIN_WRITE);
         }
     };
     return conn;

package/dist/esm/store/turso/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        await db.batch(paramRows.map((args) => ({ sql, args })));\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAOpD,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,2FAA2F;QAC3F,uLAAuL;QACvL,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBAAgBc,MAAM;gBAC1B,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD;QACF;IACF;IACA,OAAOT;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/connection.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport { BEGIN_WRITE, withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\n\n// The client's own Statement class isn't re-exported by name from '@tursodatabase/database',\n// so its type is derived structurally from Database.prepare()'s return type instead.\ntype TursoStatement = Awaited<ReturnType<Database['prepare']>>;\n\n// Turso's client is async end to end (no synchronous escape hatch like node:sqlite's), so this\n// wraps it in the same Connection/Statement shape 1:1, no sync-over-async and no worker bridge.\nclass TursoStatementWrapper implements Statement {\n  private stmt: TursoStatement;\n\n  constructor(stmt: TursoStatement) {\n    this.stmt = stmt;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    return this.stmt.run(...params);\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    return this.stmt.get(...params);\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    return this.stmt.all(...params);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    yield* this.stmt.iterate(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    return this.stmt.columns();\n  }\n\n  setReadBigInts(enabled: boolean): void {\n    this.stmt.safeIntegers(enabled);\n  }\n}\n\nexport function createConnection(db: Database): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await db.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new TursoStatementWrapper(await db.prepare(sql));\n    },\n    // db.batch() is the engine's own bulk idiom: one crossing, N rows, each its own statement,\n    // no bind-variable ceiling to chunk around. A literal nested BEGIN hard-errors, so withTransaction's join-not-savepoint helper makes this safe inside reconcile's own transaction too.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(\n        conn,\n        async () => {\n          await db.batch(paramRows.map((args) => ({ sql, args })));\n        },\n        BEGIN_WRITE\n      );\n    },\n  };\n  return conn;\n}\n"],"names":["BEGIN_WRITE","withTransaction","TursoStatementWrapper","run","params","stmt","get","all","iterate","columns","setReadBigInts","enabled","safeIntegers","createConnection","db","conn","exec","sql","prepare","runBatch","paramRows","length","batch","map","args"],"mappings":"AACA,SAASA,WAAW,EAAEC,eAAe,QAAQ,oBAAoB;AAOjE,+FAA+F;AAC/F,gGAAgG;AAChG,IAAA,AAAMC,wBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACF,GAAG,IAAIC;IAC1B;IAEA,MAAME,IAAI,GAAGF,MAAiB,EAAoB;QAChD,OAAO,IAAI,CAACC,IAAI,CAACC,GAAG,IAAIF;IAC1B;IAEA,MAAMG,IAAI,GAAGH,MAAiB,EAAsB;QAClD,OAAO,IAAI,CAACC,IAAI,CAACE,GAAG,IAAIH;IAC1B;IAEA,OAAOI,QAAQ,GAAGJ,MAAiB,EAA0B;QAC3D,OAAO,IAAI,CAACC,IAAI,CAACG,OAAO,IAAIJ;IAC9B;IAEAK,UAAmC;QACjC,OAAO,IAAI,CAACJ,IAAI,CAACI,OAAO;IAC1B;IAEAC,eAAeC,OAAgB,EAAQ;QACrC,IAAI,CAACN,IAAI,CAACO,YAAY,CAACD;IACzB;IA1BA,YAAYN,IAAoB,CAAE;QAChC,IAAI,CAACA,IAAI,GAAGA;IACd;AAyBF;AAEA,OAAO,SAASQ,iBAAiBC,EAAY;IAC3C,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,GAAGE,IAAI,CAACC;QAChB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAIf,sBAAsB,MAAMY,GAAGI,OAAO,CAACD;QACpD;QACA,2FAA2F;QAC3F,uLAAuL;QACvL,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAUC,MAAM,KAAK,GAAG;YAC5B,MAAMpB,gBACJc,MACA;gBACE,MAAMD,GAAGQ,KAAK,CAACF,UAAUG,GAAG,CAAC,CAACC,OAAU,CAAA;wBAAEP;wBAAKO;oBAAK,CAAA;YACtD,GACAxB;QAEJ;IACF;IACA,OAAOe;AACT"}

package/dist/esm/store/turso/open.d.ts

@@ -9,4 +9,6 @@ export interface OpenResult {
     parsed: number;
     warnings: string[];
 }
+export declare const CONTENT_FTS_DDL: readonly ["CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')", "CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')"];
+export declare const CONTENT_FTS_NAMES: readonly ["content_fts", "content_fts_ngram"];
 export declare function openTurso(cfg: ResolvedConfig): Promise<OpenResult>;

package/dist/esm/store/turso/open.js

@@ -17,11 +17,20 @@ export const DB_FILENAME = 'cache.turso.db';
 export const SCHEMA_VERSION = '3';
 // The ngram index is scoped to disjoint "_ngram" sidecar columns: a second index over the same
 // columns makes a bare substring match a whole word, defeating the prefix-query rejection.
+// The two FTS indexes, named here because reconcile drops and rebuilds them around a bulk load:
+// Tantivy maintains them per inserted row, which is quadratic in what is already indexed.
+export const CONTENT_FTS_DDL = [
+    `CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`,
+    `CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`
+];
+export const CONTENT_FTS_NAMES = [
+    'content_fts',
+    'content_fts_ngram'
+];
 async function ensureSchema(conn, cfg) {
     await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter ("path" TEXT PRIMARY KEY, "_mtime" REAL, "_ctime" REAL, "_size" INTEGER, "_parse_error" TEXT)`);
     await conn.exec(`CREATE TABLE IF NOT EXISTS content ("path" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT, title_ngram TEXT, summary_ngram TEXT, text_ngram TEXT)`);
-    await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts ON content USING fts (title, summary, text) WITH (weights = 'title=10.0,summary=5.0,text=1.0')`);
-    await conn.exec(`CREATE INDEX IF NOT EXISTS content_fts_ngram ON content USING fts (title_ngram, summary_ngram, text_ngram) WITH (tokenizer='ngram', weights='title_ngram=10.0,summary_ngram=5.0,text_ngram=1.0')`);
+    for (const ddl of CONTENT_FTS_DDL)await conn.exec(ddl);
     await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files ("path" TEXT, preset TEXT, PRIMARY KEY ("path", preset))`);
     await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');
     for (const feature of activeFeatures(cfg)){