sensemaking 0.19.0 → 0.19.2

package/dist/cjs/store/turso/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Fork of sqlite/reconcile.ts, not duckdb's, so this store keeps reconcile_max_ms bookkeeping\n// (open.ts's busy timeout derives from it). content is a plain table with \"_ngram\" sidecars for lexical.ts.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE, rather than the delete-and-insert `OR` clause, keeps the row's identity stable across reparses.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  const txStart = Date.now();\n  await withTransaction(conn, async () => {\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n    // content), so vanished and reparsed docs delete in one pass.\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout: the observed\n  // max is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = await getMeta(conn, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // gets recorded rather than losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) await setMeta(conn, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","txStart","durationMs","prevRaw","prevMax","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","Date","now","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile","getMeta","Number","setMeta","String"],"mappings":";;;;+BAmCsBA;;;eAAAA;;;wBAlCK;uBACI;0BAEN;wBAEmB;yBACf;yBACE;wBAC0B;6BACzB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,8FAA8F;AAC9F,4GAA4G;AAC5G,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAEhC,yFAAyF;AACzF,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEO,SAAeJ,UAAUc,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAIAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YASAC,iBAEAC,WAKAC,OACAC,OACAC,UACAC,kBAEAC,SA0DAC,YACAC,SAGAC;;;;oBA5GAxB,QAAQyB,IAAAA,mBAAS,EAAC3B,KAAKC;oBACvBE,aAAa,IAAIhB,IAAIe,MAAM0B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAEhC;;wBAAMC,KAAK+B,OAAO,CAAC;;;oBAAlC1B,eAAe;oBACC;;wBAAMA,aAAa2B,GAAG;;;oBAAtC1B,eAAgB;oBAChBC,WAAW,IAAI0B,IAAI3B,aAAauB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtD1B,WAAWF,aAAa8B,MAAM,CAAC,SAACF;+BAAM,CAAC9B,WAAWiC,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhF1B,YAAYN,MAAMiC,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM/B,SAASgC,GAAG,CAACT,EAAE/B,OAAO;wBAClC,OAAO,CAACuC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAInC,SAASoC,MAAM,KAAK,KAAKnC,UAAUmC,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG/B,QAAQ;wBAAK;;oBAEhFJ,WAAWoC,IAAAA,uBAAc,EAAC7C;oBACZ;;wBAAM8C,IAAAA,oBAAU,EAAC/C;;;oBAA/BW,cAAc;oBAEpB,oFAAoF;oBACpF,uDAAuD;oBACjDC,SAASoC,IAAAA,oBAAQ,EAAC,mBAAmBvC,UAAUmC,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACxC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOsC,IAAI;;;oBAArE,OAAA,eAArCrC,aAAqC,KAA3CsC,MAAkBrC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOwC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAY0C,GAAG,CAACrC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAW2B,MAAM,GAAGpD,yBAAyB;wBAC/C,MAAM,IAAI8D,oBAAU,CAClB,gBACA,AAAC,0BAAwG9D,OAA/EyB,WAAW2B,MAAM,EAAC,+DAAqF,OAAxBpD,yBAAwB;oBAErI;oBACA,0FAA0F;oBAC1F,sEAAsE;oBAChE0B,kBAAkBD,WAAWmB,MAAM,CAAC,SAACmB;+BAAMpE,yBAAyBkD,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBAC1G,sHAAsH;oBAChHpC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBW,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FxC,OAAhFA,gBAAgBW,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKxC,gBAClLkB,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFtC,QAAQX,UAAU2B,MAAM,CAAC,SAACN;+BAAM,CAACvB,SAAS8B,GAAG,CAACP,EAAE/B,OAAO;uBAAG8B,GAAG,CAAC,SAACC;+BAAMA,EAAE/B,OAAO;;oBAC9EsB,QAAwB;wBAAElB,OAAAA;wBAAOwD,UAAU9C,WAAWgB,GAAG,CAAC,SAAC+B;mCAAMA,EAAE7D,OAAO;;wBAAGqB,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIlC,IAAIgC;oBACnBG,mBAAmBV,WAAWgB,GAAG,CAAC,SAAC+B;+BAAMA,EAAE7D,OAAO;uBAAEqC,MAAM,CAAC,SAACyB;+BAAM,CAACvC,SAASe,GAAG,CAACwB;;oBAEhFrC,UAAUsC,KAAKC,GAAG;oBACxB;;wBAAMC,IAAAA,8BAAe,EAAChE,MAAM;;oCA+CyCiE,iBAK7BA,yBAnDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAMjD,UAGHkD,MAgBFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAM1E,KAAwB,YAAA,QAAM2E,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAnDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAalD;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKwE,IAAI,CAAC,AAAC,sCAAqD,OAAhBf,IAAAA,oBAAU,EAACzC;;;4CAAjE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CACIqD,OAAOrD,WAAWgB,GAAG,CAAC,SAAClC;uDAC3BuB,gBAAgBW,GAAG,CAAC,SAACb;wDAOZrB;oDANP,IAAIqB,QAAQ,QAAQ,OAAOrB,IAAII,OAAO;oDACtC,IAAIiB,QAAQ,UAAU,OAAOrB,IAAI8C,OAAO;oDACxC,IAAIzB,QAAQ,UAAU,OAAOrB,IAAI8E,OAAO;oDACxC,IAAIzD,QAAQ,SAAS,OAAOrB,IAAIgD,IAAI;oDACpC,mFAAmF;oDACnF,IAAI3B,QAAQ,gBAAgB,OAAOrB,IAAI+E,UAAU;oDACjD,QAAO/E,gBAAAA,IAAIgF,IAAI,CAAC3D,IAAI,cAAbrB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMK,KAAK4E,QAAQ,CAACzD,WAAW+C;;;4CAA/B;;;4CAGF,0FAA0F;4CAC1F,8DAA8D;4CACxDC,iBAAiB,AAAC,qBAAG3D,iBAAU,qBAAGe;iDACpC4C,CAAAA,eAAevB,MAAM,GAAG,CAAA,GAAxBuB;;;;4CACF;;gDAAMnE,KAAK4E,QAAQ,CACjB,wCACAT,eAAetC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIEhD,CAAAA,WAAW+B,MAAM,GAAG,CAAA,GAApB/B;;;;4CAAuB;;gDAAMb,KAAK4E,QAAQ,CAACnF,oBAAoBoB,WAAWgB,GAAG,CAACnC;;;4CAAvD;;;iDAEvBc,CAAAA,SAASoC,MAAM,GAAG,CAAA,GAAlBpC;;;;4CACF;;gDAAMR,KAAK4E,QAAQ,CACjB,4CACApE,SAASqB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DO,gBAAgB,AAAC,qBAAG5D,iBAAU,qBAAGe;iDACnC6C,CAAAA,cAAcxB,MAAM,GAAG,CAAA,GAAvBwB;;;;4CACF;;gDAAMpE,KAAK4E,QAAQ,CACjB,6CACAR,cAAcvC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIQ;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAaxD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMlB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAIkF,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAEnF,IAAII,OAAO;gEAAEuE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWzB,MAAM,GAAG,CAAA,GAApByB;;;;4CAAuB;;gDAAMrE,KAAK4E,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAG/D,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9BgD,CAAAA,aAAa3B,MAAM,GAAG,CAAA,GAAtB2B;;;;;;;;;;;;4CAA8B,aAAiB7D;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBjE,MAAMuE,cAAclD;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAM4C,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiCnE,WAAWgB,GAAG,CAAC,SAAClC;uEAAS;oEAAEwC,MAAMxC,IAAII,OAAO;oEAAEkF,WAAWtF,IAAIsF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBjE,MAAMgF,gBAAgB3D;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMuD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyBjE,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBArDA;oBAuDA,8FAA8F;oBAC9F,qDAAqD;oBAC/CI,aAAaqC,KAAKC,GAAG,KAAKvC;oBAChB;;wBAAM6D,IAAAA,iBAAO,EAACrF,MAAM;;;oBAA9B0B,UAAU;oBAChB,yFAAyF;oBACzF,0EAA0E;oBACpEC,UAAUD,YAAY,OAAO,CAAC,IAAI4D,OAAO5D;yBAC3CD,CAAAA,aAAaE,OAAM,GAAnBF;;;;oBAAsB;;wBAAM8D,IAAAA,iBAAO,EAACvF,MAAM,oBAAoBwF,OAAO/D;;;oBAA/C;;;oBAE1B;;wBAAO;4BAAEoB,QAAQhC,WAAW+B,MAAM;4BAAE9B,UAAAA;wBAAS;;;;IAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { hasUnspacedRun } from '../../text/segment.ts';\nimport { recordReconcileDuration } from '../shared.ts';\nimport { BEGIN_WRITE } from '../transaction.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { CONTENT_FTS_DDL, CONTENT_FTS_NAMES } from './open.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. content is a plain table with \"_ngram\" sidecars for lexical.ts.\n\n// '' when the field has no unspaced-script run (the common case), so the ngram index carries\n// nothing for it -- same \"pay for nothing when absent\" shape as sqlite's segmentField sidecars.\nfunction ngramSidecar(text: string): string {\n  return hasUnspacedRun(text) ? text : '';\n}\n\n// Not a compile-time cap on ALTER TABLE ADD COLUMN (spike-measured: turso accepts 10,000 with\n// no error); the real fence is a SELECT projecting more than this many result columns, which fails to prepare.\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Changed files above which rebuilding the FTS index beats maintaining it per row. Measured\n// 2026-08-30; the derivation and its bounds are pinned in this file's spec.\nexport const FTS_REBUILD_THRESHOLD = 250;\n\n// No rowid coupling (unlike sqlite's FTS5 content): `path` is content's own primary key.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text, title_ngram, summary_ngram, text_ngram) VALUES (?, ?, ?, ?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  const { title, summary, text } = doc.search;\n  return [doc.relPath, title, summary, text, ngramSidecar(title), ngramSidecar(summary), ngramSidecar(text)];\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const churn = delta.reparsed.length + delta.vanished.length;\n\n  // Tantivy indexes per inserted row at a cost that grows with the batch, so a large insert is\n  // superlinear and a rebuild wins past the threshold.\n  const bulk = delta.files.length === 0 || churn > FTS_REBUILD_THRESHOLD;\n  if (bulk) for (const name of CONTENT_FTS_NAMES) await conn.exec(`DROP INDEX IF EXISTS ${name}`);\n\n  // content is a plain table keyed by its own path (no rowid subquery, unlike sqlite's FTS5\n  // content), so vanished and reparsed docs delete in one pass.\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  if (bulk) for (const ddl of CONTENT_FTS_DDL) await conn.exec(ddl);\n}\n\nexport const tursoDialect: ReconcileDialect = {\n  beginMode: () => BEGIN_WRITE,\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError(\n        'COLUMN_LIMIT',\n        `frontmatter would need ${count} columns, crossing turso's SELECT result-set column limit (${MAX_FRONTMATTER_COLUMNS}; ALTER TABLE ADD COLUMN itself accepts far more, but a query projecting past this many columns fails to prepare). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n      );\n    }\n  },\n  columnTypeSuffix: () => '',\n  reconcileContent,\n  recordDuration: recordReconcileDuration,\n};\n"],"names":["FTS_REBUILD_THRESHOLD","tursoDialect","ngramSidecar","text","hasUnspacedRun","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","contentRow","doc","search","title","summary","relPath","reconcileContent","conn","touched","docs","delta","churn","bulk","name","ddl","reparsed","length","vanished","files","CONTENT_FTS_NAMES","exec","runBatch","map","p","CONTENT_FTS_DDL","beginMode","BEGIN_WRITE","checkColumnLimit","count","SenseError","columnTypeSuffix","recordDuration","recordReconcileDuration"],"mappings":";;;;;;;;;;;QAwBaA;eAAAA;;QA6BAC;eAAAA;;;wBArDc;yBAGI;wBACS;6BACZ;sBAEuB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEnD,qFAAqF;AACrF,sFAAsF;AAEtF,6FAA6F;AAC7F,gGAAgG;AAChG,SAASC,aAAaC,IAAY;IAChC,OAAOC,IAAAA,yBAAc,EAACD,QAAQA,OAAO;AACvC;AAEA,8FAA8F;AAC9F,+GAA+G;AAC/G,IAAME,0BAA0B;AAIzB,IAAML,wBAAwB;AAErC,yFAAyF;AACzF,IAAMM,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,IAAiCA,cAAAA,IAAIC,MAAM,EAAnCC,QAAyBF,YAAzBE,OAAOC,UAAkBH,YAAlBG,SAASR,OAASK,YAATL;IACxB,OAAO;QAACK,IAAII,OAAO;QAAEF;QAAOC;QAASR;QAAMD,aAAaQ;QAAQR,aAAaS;QAAUT,aAAaC;KAAM;AAC5G;AAEA,SAAeU,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OAIAC,MACS,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,WAUN,4BAAA,oBAAA,iBAAA,YAAA,QAAMC;;;;oBAffH,QAAQD,MAAMK,QAAQ,CAACC,MAAM,GAAGN,MAAMO,QAAQ,CAACD,MAAM;oBAE3D,6FAA6F;oBAC7F,qDAAqD;oBAC/CJ,OAAOF,MAAMQ,KAAK,CAACF,MAAM,KAAK,KAAKL,QAAQlB;oBAClC,kCAAA,2BAAA;yBAAXmB,MAAAA;;;;;;;;;;;;oBAAW,YAAcO,yBAAiB;;;2BAA/B,6BAAA,QAAA;;;;oBAAMN,OAAN;oBAAiC;;wBAAMN,KAAKa,IAAI,CAAC,AAAC,wBAA4B,OAALP;;;oBAAxC;;;oBAAjC;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;yBAIXL,CAAAA,QAAQQ,MAAM,GAAG,CAAA,GAAjBR;;;;oBACF;;wBAAMD,KAAKc,QAAQ,CACjB,wCACAb,QAAQc,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEd,CAAAA,KAAKO,MAAM,GAAG,CAAA,GAAdP;;;;oBAAiB;;wBAAMF,KAAKc,QAAQ,CAACtB,oBAAoBU,KAAKa,GAAG,CAACtB;;;oBAAjD;;;oBACN,mCAAA,4BAAA;yBAAXY,MAAAA;;;;;;;;;;;;oBAAW,aAAaY,uBAAe;;;2BAA5B,8BAAA,SAAA;;;;oBAAMV,MAAN;oBAA8B;;wBAAMP,KAAKa,IAAI,CAACN;;;oBAAhB;;;oBAA9B;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;;;;;;IACjB;;AAEO,IAAMpB,eAAiC;IAC5C+B,WAAW,SAAXA;eAAiBC,0BAAW;;IAC5BC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQ9B,yBAAyB;YACnC,MAAM,IAAI+B,oBAAU,CAClB,gBACA,AAAC,0BAA4F/B,OAAnE8B,OAAM,+DAAqF,OAAxB9B,yBAAwB;QAEzH;IACF;IACAgC,kBAAkB,SAAlBA;eAAwB;;IACxBxB,kBAAAA;IACAyB,gBAAgBC,iCAAuB;AACzC"}

package/dist/cjs/store/turso/store.js

@@ -17,12 +17,13 @@ _export(exports, {
     }
 });
 var _typests = require("../../embed/types.js");
+var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets = require("./reconcile.js");
+var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -432,7 +433,7 @@ function createStore(db, conn, cfg, baseDir) {
                 return _ts_generator(this, function(_state) {
                     return [
                         2,
-                        (0, _reconcilets.reconcile)(conn, cfg, baseDir)
+                        (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.tursoDialect)
                     ];
                 });
             })();

package/dist/cjs/store/turso/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":";;;;;;;;;;;QAcaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACf;yBACE;2BACH;0BACoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC;;;YAC9B;;QACAY,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAAChB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAiB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACjB,MAAMe,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACvB,MAAMqB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAAC1B;;YAC3B2B,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC7B,MAAM8B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACpC,MAAMgC,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACvC,MAAM8B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAACzC,MAAMsC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAM5C,GAAGQ,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCqC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGvC,SAAN,SAAMA,QAAQD,GAAW;;wBACjByC;;;;gCAAO;;oCAAMhD,GAAGQ,OAAO,CAACD;;;gCAAxByC,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMpD,GAAGoD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/turso/store.ts"],"sourcesContent":["import type { Database } from '@tursodatabase/database';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { queryLexical } from './lexical.ts';\nimport { tursoDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// No 'snippets': fts_highlight returns the whole column, not a bounded window, so hits use\n// the caller's JS excerpt. No 'watch-concurrency': single-process by default, as on duckdb.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere.\nexport function createStore(db: Database, conn: Connection, cfg: Config, baseDir: string): Store {\n  return {\n    name: 'turso',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      return reconcile(conn, cfg, baseDir, tursoDialect);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: (terms, opts) => queryLexical(conn, terms, opts),\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(conn, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    async engineStatus() {\n      // Read back rather than recomputed: this is what open() actually set (3x the largest\n      // recorded reconcile, floored at 30s, capped at 10min). Turso's PRAGMA busy_timeout names its column \"busy_timeout\" (spike-verified), not \"timeout\" like real SQLite.\n      const row = (await (await db.prepare('PRAGMA busy_timeout')).get()) as { busy_timeout: number };\n      return { busy_timeout: `${row.busy_timeout}ms (derived: 3x the largest reconcile this cache has recorded, floored at 30000ms)` };\n    },\n    raw: {\n      async prepare(sql: string) {\n        const stmt = await db.prepare(sql);\n        stmt.safeIntegers(true); // int64 past 2^53 arrives as BigInt instead of losing precision\n        return {\n          columns: () => stmt.columns(),\n          // sense sql streams through the client's own async generator.\n          iterate: async function* (...params: unknown[]) {\n            yield* stmt.iterate(...params);\n          },\n        };\n      },\n    },\n    async close() {\n      await db.close();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","db","conn","cfg","baseDir","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","tursoDialect","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","terms","opts","queryLexical","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","scanSimilar","hasVector","hasVectorRow","engineStatus","row","get","busy_timeout","raw","stmt","safeIntegers","iterate","params","close"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIGC;eAAAA;;;uBAjBW;2BACD;wBACC;6BACK;yBAEU;4BACf;yBACE;4BACA;0BACiC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,EAAY,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtF,OAAO;QACLC,MAAM;QACNC,cAAcR;QACRS,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACME,WAAN,SAAMA;;;oBACJ;;wBAAOA,IAAAA,sBAAS,EAACb,MAAMC,KAAKC,SAASY,0BAAY;;;YACnD;;QACAC,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACjB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAkB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAClB,MAAMgB,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAO,SAAPA,MAAQC,OAAOC;uBAASC,IAAAA,uBAAY,EAACxB,MAAMsB,OAAOC;;QACpD;QACA,sFAAsF;QACtF,gIAAgI;QAChIE,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAAC3B;;YAC3B4B,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC9B,MAAM+B,mBAAU,EAAEF;;YAC3DG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACrC,MAAMiC,IAAIF,mBAAU,EAAEI,OAAOC;;YAC5FE,SAAS,SAATA,QAAUC,MAAMhB;uBAASiB,IAAAA,uBAAW,EAACxC,MAAM+B,mBAAU,EAAEQ,MAAMhB;;YAC7DkB,WAAW,SAAXA,UAAYF;uBAASG,IAAAA,uBAAY,EAAC1C,MAAMuC;;QAC1C;QACMI,cAAN,SAAMA;;oBAGEC;;;;4BAAc;;gCAAM7C,GAAGQ,OAAO,CAAC;;;4BAAxB;;gCAAO,cAAyCsC,GAAG;;;4BAA1DD,MAAO;4BACb;;gCAAO;oCAAEE,cAAc,AAAC,GAAmB,OAAjBF,IAAIE,YAAY,EAAC;gCAAoF;;;;YACjI;;QACAC,KAAK;YACGxC,SAAN,SAAMA,QAAQD,GAAW;;wBACjB0C;;;;gCAAO;;oCAAMjD,GAAGQ,OAAO,CAACD;;;gCAAxB0C,OAAO;gCACbA,KAAKC,YAAY,CAAC,OAAO,gEAAgE;gCACzF;;oCAAO;wCACLjC,SAAS,SAATA;mDAAegC,KAAKhC,OAAO;;wCAC3B,8DAA8D;wCAC9DkC,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDACrCH;;;;4DAAP;;2EAAA,0CAAOA,CAAAA,QAAAA,MAAKE,OAAO,OAAZF,OAAa,qBAAGG;;;4DAAvB;;;;;;4CACF;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;;;4BACJ;;gCAAMrD,GAAGqD,KAAK;;;4BAAd;;;;;;YACF;;IACF;AACF"}

package/dist/cjs/store/types.d.cts

@@ -1,5 +1,8 @@
-import type { StoreName } from '../config/index.js';
-export type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';
+import type { Config, StoreName } from '../config/index.js';
+import type { ReconcileDelta } from '../features/types.js';
+import type { ParsedDoc } from '../scan/index.js';
+export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
+export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
     lastInsertRowid: number | bigint;
@@ -19,6 +22,13 @@ export interface Connection {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
 }
+export interface ReconcileDialect {
+    beginMode(): string;
+    checkColumnLimit(count: number): void;
+    columnTypeSuffix(): string;
+    reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
+    recordDuration?(conn: Connection, ms: number): Promise<void>;
+}
 export interface FieldStat {
     field: string;
     coverage: number;

package/dist/cjs/store/types.d.ts

@@ -1,5 +1,8 @@
-import type { StoreName } from '../config/index.js';
-export type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';
+import type { Config, StoreName } from '../config/index.js';
+import type { ReconcileDelta } from '../features/types.js';
+import type { ParsedDoc } from '../scan/index.js';
+export declare const CAPABILITY_NAMES: readonly ["phrases", "snippets", "watch-concurrency", "lexical", "vectors", "sql-functions"];
+export type Capability = (typeof CAPABILITY_NAMES)[number];
 export interface RunResult {
     changes: number | bigint;
     lastInsertRowid: number | bigint;
@@ -19,6 +22,13 @@ export interface Connection {
     prepare(sql: string): Promise<Statement>;
     runBatch(sql: string, paramRows: unknown[][]): Promise<void>;
 }
+export interface ReconcileDialect {
+    beginMode(): string;
+    checkColumnLimit(count: number): void;
+    columnTypeSuffix(): string;
+    reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;
+    recordDuration?(conn: Connection, ms: number): Promise<void>;
+}
 export interface FieldStat {
     field: string;
     coverage: number;

package/dist/cjs/store/types.js

@@ -4,4 +4,18 @@
 Object.defineProperty(exports, "__esModule", {
     value: true
 });
+Object.defineProperty(exports, "CAPABILITY_NAMES", {
+    enumerable: true,
+    get: function() {
+        return CAPABILITY_NAMES;
+    }
+});
+var CAPABILITY_NAMES = [
+    'phrases',
+    'snippets',
+    'watch-concurrency',
+    'lexical',
+    'vectors',
+    'sql-functions'
+];
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { StoreName } from '../config/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. The rest are finer FTS5-only behaviors; a missing one fails at open or first use.\nexport type Capability = 'phrases' | 'snippets' | 'watch-concurrency' | 'lexical' | 'vectors';\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":[],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/types.ts"],"sourcesContent":["// The backing-store interface: a minimal portable statement surface (exec/prepare), plus\n// dedicated interfaces exactly where engines diverge (lexical index, vector scan, raw sql).\n\nimport type { Config, StoreName } from '../config/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\n\n// 'lexical'/'vectors': the store's LexicalIndex/VectorStore is functionally implemented, not\n// present-but-inert. 'sql-functions': the engine can register has/basename/segment as UDFs at all,\n// which turso's client cannot. The rest are finer FTS5-only behaviors; a missing one fails at open\n// or first use. An array, not a bare union, so a runtime check reads the same list the type does.\nexport const CAPABILITY_NAMES = ['phrases', 'snippets', 'watch-concurrency', 'lexical', 'vectors', 'sql-functions'] as const;\nexport type Capability = (typeof CAPABILITY_NAMES)[number];\n\nexport interface RunResult {\n  changes: number | bigint;\n  lastInsertRowid: number | bigint;\n}\n\n// A prepared statement's async surface: every supported engine crosses a real async boundary\n// for a query (DuckDB has no synchronous client), so run/get/all return Promises.\nexport interface Statement {\n  run(...params: unknown[]): Promise<RunResult>;\n  get(...params: unknown[]): Promise<unknown>;\n  all(...params: unknown[]): Promise<unknown[]>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n  columns(): Array<{ name: string }>;\n  setReadBigInts(enabled: boolean): void;\n}\n\n// Connection surface feature-owned SQL runs against inside a store's hot loops (schema,\n// reconcile), portable so a feature never imports an engine-specific client type.\nexport interface Connection {\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n}\n\n// One reconcile algorithm (src/store/reconcile.ts), parameterised per engine. reconcileContent is\n// the load-bearing member: it lands every content-table change and owns its own multi-step\n// strategy (sqlite FTS5 incremental, duckdb combined delete/insert, turso incremental-or-rebuild).\nexport interface ReconcileDialect {\n  // BEGIN mode for the whole reconcile: sqlite/turso 'BEGIN IMMEDIATE', duckdb 'BEGIN'.\n  beginMode(): string;\n  // Throws SenseError('COLUMN_LIMIT', ...) past this store's own column ceiling and reasoning.\n  checkColumnLimit(count: number): void;\n  // Type suffix for ALTER TABLE ADD COLUMN: '' for sqlite/turso, ' VARIANT' for duckdb.\n  columnTypeSuffix(): string;\n  // Deletes content rows for `touched`, inserts rows for `docs`; `delta` carries the tree state a\n  // strategy may need. Must not open its own transaction, and must not return before its own\n  // multi-step strategy (e.g. turso's DROP/rebuild) completes.\n  reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta, cfg: Config): Promise<void>;\n  // Records this reconcile's write-transaction duration. sqlite/turso use it for open()'s derived\n  // busy_timeout; duckdb has no such PRAGMA and omits it.\n  recordDuration?(conn: Connection, ms: number): Promise<void>;\n}\n\nexport interface FieldStat {\n  field: string;\n  coverage: number;\n  type: string;\n}\n\nexport interface DocumentStore {\n  // Frontmatter column names (including internal ones; callers filter).\n  columns(): Promise<string[]>;\n  // Per-column coverage (non-null count) and observed type set, aggregated in one SQL query,\n  // never one row per note. `scopeWhere` is a caller-built WHERE fragment, per LexicalQueryOptions.\n  fieldStats(columns: string[], scopeWhere: string): Promise<FieldStat[]>;\n}\n\nexport interface LexicalHit {\n  path: string;\n  hit: string | null;\n}\n\nexport interface LexicalQueryOptions {\n  whereJoin: string;\n  whereCond: string;\n  scopeCond: string;\n  limit: number;\n}\n\nexport interface LexicalIndex {\n  // Ranked word-match query with excerpt, scoped by the caller-built SQL fragments (the same\n  // fragments narrowByWhere/materializeScope produce elsewhere).\n  query(terms: string, opts: LexicalQueryOptions): Promise<LexicalHit[]>;\n}\n\nexport interface VectorCandidate {\n  path: string;\n  lines: string;\n  similarity: number;\n}\n\nexport interface VectorSimilar {\n  path: string;\n  similarity: number;\n}\n\nexport interface VectorWriteRow {\n  path: string;\n  chunk: number;\n  scale: number;\n  vector: Buffer;\n}\n\nexport interface VectorStore {\n  // Rows whose vector is NULL: never embedded, or added since.\n  pending(): Promise<Array<{ path: string; chunk: number }>>;\n  // One batch write per call (never per row) so a provider's embedding batch stays inside a\n  // single store method.\n  writeVectors(rows: VectorWriteRow[]): Promise<void>;\n  candidates(queryVector: Float32Array, storeDims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;\n  similar(path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]>;\n  hasVector(path: string): Promise<boolean>;\n}\n\n// The `sense sql` passthrough: string in, streamed rows out. Each store registers the same\n// sense-supplied functions and applies its own read-bigints/error-translation behavior.\nexport interface RawStatement {\n  columns(): Array<{ name: string }>;\n  iterate(...params: unknown[]): AsyncIterable<unknown>;\n}\n\nexport interface SqlSession {\n  prepare(sql: string): Promise<RawStatement>;\n}\n\nexport interface Store {\n  readonly name: StoreName;\n  readonly capabilities: ReadonlySet<Capability>;\n  exec(sql: string): Promise<void>;\n  prepare(sql: string): Promise<Statement>;\n  // One crossing, N rows: every external write loop (search's candidate insert, graph ring's\n  // temp-table writes) goes through this instead of looping over `run()`. Same contract as Connection.runBatch.\n  runBatch(sql: string, paramRows: unknown[][]): Promise<void>;\n  // Pins one snapshot across multi-statement reads; `map` and `peek` use it. A network-bound\n  // write (search's embedding top-up) must stay outside it. Nesting joins the enclosing transaction.\n  transaction<T>(fn: () => Promise<T>): Promise<T>;\n  // The whole file-sync pass (parse changed files, run every feature hook, resolve links,\n  // recompute rank) behind one method: per-file iteration never crosses the async boundary.\n  reconcile(): Promise<{ parsed: number; warnings: string[] }>;\n  docs: DocumentStore;\n  lexical: LexicalIndex;\n  vectors: VectorStore;\n  raw: SqlSession;\n  // Engine-level facts for `sense status` (e.g. a derived busy_timeout PRAGMA reading); each\n  // store owns what it reports and how it is worded. Empty when there is nothing to report.\n  engineStatus(): Promise<Record<string, string>>;\n  close(): Promise<void>;\n}\n"],"names":["CAPABILITY_NAMES"],"mappings":"AAAA,yFAAyF;AACzF,4FAA4F;;;;;+BAU/EA;;;eAAAA;;;AAAN,IAAMA,mBAAmB;IAAC;IAAW;IAAY;IAAqB;IAAW;IAAW;CAAgB"}

package/dist/esm/index.d.ts

@@ -7,7 +7,7 @@ export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
 export type { OpenResult } from './store/index.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js

@@ -4,5 +4,5 @@ export { mapTree, peek, presetCoverage, search } from './commands/index.js';
 export { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.js';
 export { SenseError } from './errors.js';
 export { printRows } from './output/output.js';
-export { clearCache, openStore as open } from './store/index.js';
+export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export { runWatch } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIjE,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAInF,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/store/duckdb/fieldStats.js

@@ -7,25 +7,38 @@ function classifyVariantType(name, field) {
     if (name === 'VARCHAR') return 'text';
     throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
 }
-// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
+// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in
+// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same
+// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:
+// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking
+// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no
+// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300
+// columns runs in 7ms.
+const AGGREGATES_PER_QUERY = 16;
+// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
 // '|' is the separator, not ',': DECIMAL's display name ("DECIMAL(4, 2)") contains a comma.
 export async function fieldStats(conn, columns, scopeWhere) {
-    const exprs = columns.map((name, i)=>{
-        const quoted = quoteIdent(name);
-        return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
-    });
-    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
-    const row = await stmt.get();
-    return columns.map((name, i)=>{
-        var _row_, _row_1;
-        const raw = ((_row_ = row[`t${i}`]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
-        const types = [
-            ...new Set(raw.map((t)=>classifyVariantType(t, name)))
-        ].sort();
-        return {
-            field: name,
-            coverage: Number((_row_1 = row[`n${i}`]) !== null && _row_1 !== void 0 ? _row_1 : 0),
-            type: types.join(',')
-        };
-    });
+    const stats = [];
+    for(let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY){
+        const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);
+        const exprs = chunk.map((name, i)=>{
+            const quoted = quoteIdent(name);
+            return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
+        });
+        const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
+        const row = await stmt.get();
+        for (const [i, name] of chunk.entries()){
+            var _row_, _row_1;
+            const raw = ((_row_ = row[`t${i}`]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
+            const types = [
+                ...new Set(raw.map((t)=>classifyVariantType(t, name)))
+            ].sort();
+            stats.push({
+                field: name,
+                coverage: Number((_row_1 = row[`n${i}`]) !== null && _row_1 !== void 0 ? _row_1 : 0),
+                type: types.join(',')
+            });
+        }
+    }
+    return stats;
 }

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","fieldStats","conn","columns","scopeWhere","exprs","map","i","quoted","stmt","prepare","join","row","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,8IAA8I;AAC9I,4FAA4F;AAC5F,OAAO,eAAeI,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACV,MAAMW;QAC/B,MAAMC,SAASd,WAAWE;QAC1B,OAAO,CAAC,MAAM,EAAEY,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IACvI;IACA,MAAME,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEP,YAAY;IAC3F,MAAMQ,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOV,QAAQG,GAAG,CAAC,CAACV,MAAMW;YACVK,OAEyBA;QAFvC,MAAME,MAAM,EAAEF,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAIG,KAAK,CAAC,KAAKC,MAAM,CAACC;QAC/D,MAAMC,QAAQ;eAAI,IAAIC,IAAIL,IAAIR,GAAG,CAAC,CAACc,IAAMzB,oBAAoByB,GAAGxB;SAAQ,CAACyB,IAAI;QAC7E,OAAO;YAAExB,OAAOD;YAAM0B,UAAUC,QAAOX,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;YAAIY,MAAMN,MAAMP,IAAI,CAAC;QAAK;IACnF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// duckdb degrades superlinearly in the number of string_agg(DISTINCT variant_typeof()) aggregates in\n// one projection: 301 columns cost 1148ms in one query against 10ms for COUNT alone over the same\n// columns, so it is the aggregate count, not the width. Measured chunk sizes at 301 columns:\n// 10 -> 71ms, 16 -> 81ms, 25 -> 88ms, 40 -> 110ms, 64 -> 158ms, flat between 10 and 25. Chunking\n// also wins on an ordinary tree (31 columns: 17ms in one query, 11ms chunked), so there is no\n// small-tree cost to trade against. sqlite and turso need none of this: GROUP_CONCAT over 300\n// columns runs in 7ms.\nconst AGGREGATES_PER_QUERY = 16;\n\n// COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const stats: FieldStat[] = [];\n  for (let start = 0; start < columns.length; start += AGGREGATES_PER_QUERY) {\n    const chunk = columns.slice(start, start + AGGREGATES_PER_QUERY);\n    const exprs = chunk.map((name, i) => {\n      const quoted = quoteIdent(name);\n      return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n    });\n    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n    const row = (await stmt.get()) as Record<string, number | string | null>;\n    for (const [i, name] of chunk.entries()) {\n      const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n      const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n      stats.push({ field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') });\n    }\n  }\n  return stats;\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","AGGREGATES_PER_QUERY","fieldStats","conn","columns","scopeWhere","stats","start","length","chunk","slice","exprs","map","i","quoted","stmt","prepare","join","row","get","entries","raw","split","filter","Boolean","types","Set","t","sort","push","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,qGAAqG;AACrG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,8FAA8F;AAC9F,8FAA8F;AAC9F,uBAAuB;AACvB,MAAMI,uBAAuB;AAE7B,yGAAyG;AACzG,4FAA4F;AAC5F,OAAO,eAAeC,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAqB,EAAE;IAC7B,IAAK,IAAIC,QAAQ,GAAGA,QAAQH,QAAQI,MAAM,EAAED,SAASN,qBAAsB;QACzE,MAAMQ,QAAQL,QAAQM,KAAK,CAACH,OAAOA,QAAQN;QAC3C,MAAMU,QAAQF,MAAMG,GAAG,CAAC,CAAChB,MAAMiB;YAC7B,MAAMC,SAASpB,WAAWE;YAC1B,OAAO,CAAC,MAAM,EAAEkB,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;QACvI;QACA,MAAME,OAAO,MAAMZ,KAAKa,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEZ,YAAY;QAC3F,MAAMa,MAAO,MAAMH,KAAKI,GAAG;QAC3B,KAAK,MAAM,CAACN,GAAGjB,KAAK,IAAIa,MAAMW,OAAO,GAAI;gBACzBF,OAE6BA;YAF3C,MAAMG,MAAM,EAAEH,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAII,KAAK,CAAC,KAAKC,MAAM,CAACC;YAC/D,MAAMC,QAAQ;mBAAI,IAAIC,IAAIL,IAAIT,GAAG,CAAC,CAACe,IAAMhC,oBAAoBgC,GAAG/B;aAAQ,CAACgC,IAAI;YAC7EtB,MAAMuB,IAAI,CAAC;gBAAEhC,OAAOD;gBAAMkC,UAAUC,QAAOb,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;gBAAIc,MAAMP,MAAMR,IAAI,CAAC;YAAK;QACvF;IACF;IACA,OAAOX;AACT"}

package/dist/esm/store/duckdb/open.js

@@ -6,10 +6,11 @@ import { STORE_DIMS } from '../../embed/types.js';
 import { SenseError } from '../../errors.js';
 import { activeFeatures, FEATURES } from '../../features/index.js';
 import { clearCache } from '../cache.js';
+import { reconcile } from '../reconcile.js';
 import { getMeta, setMeta } from '../shared.js';
 import { createConnection } from './connection.js';
 import { DUCKDB_PACKAGE, duckdbApi } from './native.js';
-import { reconcile } from './reconcile.js';
+import { duckdbDialect } from './reconcile.js';
 import { registerFunctions } from './sql-functions.js';
 import { createStore } from './store.js';
 export const DB_FILENAME = 'cache.duckdb';
@@ -87,7 +88,7 @@ async function connect(cfg) {
             return connect(cfg);
         }
         await ensureSchema(conn, cfg);
-        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);
+        const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);
         return {
             instance,
             duckdb,

package/dist/esm/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { reconcile } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","reconcile","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,8LAA8L;AAC9L,2JAA2J;AAC3J,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWjB,eAAee,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAElB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMmB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYnB,iBAAiBoB,KAAKd;AACxG;AAEA,eAAemB,QAAQL,GAAmB;IACxC,MAAMM,WAAW3B,KAAKqB,IAAIO,OAAO,EAAE1B;IACnCH,UAAU4B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS9B,KAAK2B,UAAUV;IAE9B,gIAAgI;IAChI,qLAAqL;IACrL,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe3B,YAAY,MAAM2B;QACrC,MAAM,IAAI3B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEO,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIjC,WAAW,4BAA4B,CAAC,8BAA8B,EAAEyB,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAM/B,QAAQW,MAAM;QACpC,MAAMqB,WAAW,MAAMhC,QAAQW,MAAM;QACrC,MAAMsB,eAAezC,iBAAiBoB,KAAKd;QAC3C,IAAI,AAACiC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB9B,WAAWa;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMlC,UAAUM,MAAMC,KAAKA,IAAIO,OAAO;QACnE,OAAO;YAAEO;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,MAAM8B,QAAQnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;IAClF,0FAA0F;IAC1FzB,eAAeiB,MAAM+B;IACrB,OAAO;QAAEA,OAAOA;QAAO9B,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import { mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature, STATE_DIR } from '../../config/index.ts';\nimport { rekeyChunkText } from '../../embed/handoff.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport { clearCache } from '../cache.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, Store } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '2';\n\nexport interface OpenResult {\n  store: Store;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\nasync function connect(cfg: ResolvedConfig): Promise<{ instance: DuckDBInstance; duckdb: DuckDBConnection; conn: Connection; cfg: ResolvedConfig; dbPath: string; parsed: number; warnings: string[] }> {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below (schema, reconcile's COLUMN_LIMIT) would leak the open instance, whose WAL\n  // then locks the .duckdb file undeletable on Windows. closed marks the rebuild branch.\n  let closed = false;\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n\n    await conn.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n    const version = await getMeta(conn, 'schema_version');\n    const features = await getMeta(conn, 'features');\n    const wantFeatures = featureSignature(cfg, FEATURES);\n    if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n      // Reconcile only reparses changed files, so a stale cache can't be patched incrementally --\n      // rebuild instead, same as sqlite's open() (see its comment for why this is announced).\n      const reason = version !== null && version !== SCHEMA_VERSION ? 'cache format changed (new sensemaking version)' : 'config change (features, tokenizer, or presets)';\n      console.error(`sense: ${reason}; rebuilding the index`);\n      // Close before clearCache deletes the files underneath the still-open instance.\n      closed = true;\n      duckdb.disconnectSync();\n      instance.closeSync();\n      clearCache(cfg);\n      return connect(cfg);\n    }\n\n    await ensureSchema(conn, cfg);\n\n    const { parsed, warnings } = await reconcile(conn, cfg, cfg.baseDir, duckdbDialect);\n    return { instance, duckdb, conn, cfg, dbPath, parsed, warnings };\n  } catch (err) {\n    if (!closed) {\n      duckdb.disconnectSync();\n      instance?.closeSync();\n    }\n    throw err;\n  }\n}\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  const { instance, duckdb, conn, cfg: resolvedCfg, dbPath, parsed, warnings } = await connect(cfg);\n  const store = createStore(instance, duckdb, conn, resolvedCfg, resolvedCfg.baseDir);\n  // reconcile ran before this object existed, so its chunk text is keyed by the connection.\n  rekeyChunkText(conn, store);\n  return { store: store, cfg: resolvedCfg, dbPath, parsed, warnings };\n}\n"],"names":["mkdirSync","join","featureSignature","STATE_DIR","rekeyChunkText","STORE_DIMS","SenseError","activeFeatures","FEATURES","clearCache","reconcile","getMeta","setMeta","createConnection","DUCKDB_PACKAGE","duckdbApi","duckdbDialect","registerFunctions","createStore","DB_FILENAME","SCHEMA_VERSION","ensureSchema","conn","cfg","exec","feature","name","schema","connect","stateDir","baseDir","recursive","dbPath","DuckDBInstance","err","message","duckdb","instance","create","storage_compatibility_version","closeSync","closed","version","features","wantFeatures","reason","console","error","disconnectSync","parsed","warnings","openDuckdb","resolvedCfg","store"],"mappings":"AAAA,SAASA,SAAS,QAAQ,UAAU;AACpC,SAASC,IAAI,QAAQ,YAAY;AAGjC,SAASC,gBAAgB,EAAEC,SAAS,QAAQ,wBAAwB;AACpE,SAASC,cAAc,QAAQ,yBAAyB;AACxD,SAASC,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,EAAEC,QAAQ,QAAQ,0BAA0B;AACnE,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,OAAO,EAAEC,OAAO,QAAQ,eAAe;AAEhD,SAASC,gBAAgB,QAAQ,kBAAkB;AACnD,SAASC,cAAc,EAAEC,SAAS,QAAQ,cAAc;AACxD,SAASC,aAAa,QAAQ,iBAAiB;AAC/C,SAASC,iBAAiB,QAAQ,qBAAqB;AACvD,SAASC,WAAW,QAAQ,aAAa;AAEzC,OAAO,MAAMC,cAAc,eAAe;AAC1C,qIAAqI;AACrI,iJAAiJ;AACjJ,OAAO,MAAMC,iBAAiB,IAAI;AAUlC,8LAA8L;AAC9L,2JAA2J;AAC3J,eAAeC,aAAaC,IAAgB,EAAEC,GAAW;IACvD,MAAMD,KAAKE,IAAI,CAAC,CAAC,wIAAwI,CAAC;IAC1J,MAAMF,KAAKE,IAAI,CAAC,CAAC,iGAAiG,CAAC;IACnH,MAAMF,KAAKE,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAClH,MAAMF,KAAKE,IAAI,CAAC;IAChB,KAAK,MAAMC,WAAWlB,eAAegB,KAAM;QACzC,+HAA+H;QAC/H,sHAAsH;QACtH,IAAIE,QAAQC,IAAI,KAAK,SAAS;YAC5B,MAAMJ,KAAKE,IAAI,CAAC,CAAC,kIAAkI,EAAEnB,WAAW,+BAA+B,CAAC;YAChM;QACF;QACA,MAAMoB,QAAQE,MAAM,CAACL;IACvB;IACA,IAAI,AAAC,MAAMX,QAAQW,MAAM,sBAAuB,MAAM,MAAMV,QAAQU,MAAM,kBAAkBF;IAC5F,IAAI,AAAC,MAAMT,QAAQW,MAAM,gBAAiB,MAAM,MAAMV,QAAQU,MAAM,YAAYpB,iBAAiBqB,KAAKf;AACxG;AAEA,eAAeoB,QAAQL,GAAmB;IACxC,MAAMM,WAAW5B,KAAKsB,IAAIO,OAAO,EAAE3B;IACnCH,UAAU6B,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS/B,KAAK4B,UAAUV;IAE9B,gIAAgI;IAChI,qLAAqL;IACrL,IAAIc;IACJ,IAAI;QACD,CAAA,EAAEA,cAAc,EAAE,GAAG,MAAMlB,WAAU;IACxC,EAAE,OAAOmB,KAAK;QACZ,IAAIA,eAAe5B,YAAY,MAAM4B;QACrC,MAAM,IAAI5B,WAAW,4BAA4B,CAAC,yBAAyB,EAAEQ,eAAe,UAAU,EAAE,AAACoB,IAAcC,OAAO,CAAC,CAAC,CAAC;IACnI;IAEA,IAAIC;IACJ,IAAIC;IACJ,IAAI;QACF,4HAA4H;QAC5H,4KAA4K;QAC5KA,WAAW,MAAMJ,eAAeK,MAAM,CAACN,QAAQ;YAAEO,+BAA+B;QAAS;QACzFH,SAAS,MAAMC,SAAST,OAAO;IACjC,EAAE,OAAOM,KAAK;QACZ,wFAAwF;QACxFG,qBAAAA,+BAAAA,SAAUG,SAAS;QACnB,MAAM,IAAIlC,WAAW,4BAA4B,CAAC,8BAA8B,EAAE0B,OAAO,EAAE,EAAE,AAACE,IAAcC,OAAO,EAAE;IACvH;IACA,2FAA2F;IAC3F,uFAAuF;IACvF,IAAIM,SAAS;IACb,IAAI;QACF,MAAMxB,kBAAkBmB;QACxB,MAAMd,OAAOT,iBAAiBuB;QAE9B,MAAMd,KAAKE,IAAI,CAAC;QAEhB,MAAMkB,UAAU,MAAM/B,QAAQW,MAAM;QACpC,MAAMqB,WAAW,MAAMhC,QAAQW,MAAM;QACrC,MAAMsB,eAAe1C,iBAAiBqB,KAAKf;QAC3C,IAAI,AAACkC,YAAY,QAAQA,YAAYtB,kBAAoBuB,aAAa,QAAQA,aAAaC,cAAe;YACxG,4FAA4F;YAC5F,wFAAwF;YACxF,MAAMC,SAASH,YAAY,QAAQA,YAAYtB,iBAAiB,mDAAmD;YACnH0B,QAAQC,KAAK,CAAC,CAAC,OAAO,EAAEF,OAAO,sBAAsB,CAAC;YACtD,gFAAgF;YAChFJ,SAAS;YACTL,OAAOY,cAAc;YACrBX,SAASG,SAAS;YAClB/B,WAAWc;YACX,OAAOK,QAAQL;QACjB;QAEA,MAAMF,aAAaC,MAAMC;QAEzB,MAAM,EAAE0B,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMxC,UAAUY,MAAMC,KAAKA,IAAIO,OAAO,EAAEd;QACrE,OAAO;YAAEqB;YAAUD;YAAQd;YAAMC;YAAKS;YAAQiB;YAAQC;QAAS;IACjE,EAAE,OAAOhB,KAAK;QACZ,IAAI,CAACO,QAAQ;YACXL,OAAOY,cAAc;YACrBX,qBAAAA,+BAAAA,SAAUG,SAAS;QACrB;QACA,MAAMN;IACR;AACF;AAEA,OAAO,eAAeiB,WAAW5B,GAAmB;IAClD,MAAM,EAAEc,QAAQ,EAAED,MAAM,EAAEd,IAAI,EAAEC,KAAK6B,WAAW,EAAEpB,MAAM,EAAEiB,MAAM,EAAEC,QAAQ,EAAE,GAAG,MAAMtB,QAAQL;IAC7F,MAAM8B,QAAQnC,YAAYmB,UAAUD,QAAQd,MAAM8B,aAAaA,YAAYtB,OAAO;IAClF,0FAA0F;IAC1F1B,eAAekB,MAAM+B;IACrB,OAAO;QAAEA,OAAOA;QAAO9B,KAAK6B;QAAapB;QAAQiB;QAAQC;IAAS;AACpE"}

package/dist/esm/store/duckdb/reconcile.d.ts

@@ -1,6 +1,2 @@
-import type { Config } from '../../config/index.js';
-import type { Connection } from '../types.js';
-export declare function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{
-    parsed: number;
-    warnings: string[];
-}>;
+import type { ReconcileDialect } from '../types.js';
+export declare const duckdbDialect: ReconcileDialect;

package/dist/esm/store/duckdb/reconcile.js

@@ -1,19 +1,8 @@
 import { SenseError } from '../../errors.js';
-import { activeFeatures } from '../../features/index.js';
-import { progress } from '../../output/progress.js';
-import { listFiles, RESERVED_COLUMNS } from '../../scan/index.js';
-import { reparseFiles } from '../../scan/reparse.js';
-import { getColumns, quoteIdent } from '../shared.js';
-import { withTransaction } from '../transaction.js';
-// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is
-// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
-const CORE_FRONTMATTER_COLUMNS = new Set([
-    'path',
-    '_mtime',
-    '_ctime',
-    '_size',
-    '_parse_error'
-]);
+// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
+// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
+// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
+// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).
 // No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):
 // `path` is content's own primary key, so this is a plain per-doc row.
 const INSERT_CONTENT_SQL = `INSERT INTO content ("path", title, summary, text) VALUES (?, ?, ?, ?)`;
@@ -28,107 +17,21 @@ function contentRow(doc) {
 // No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway
 // so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.
 const MAX_FRONTMATTER_COLUMNS = 10000;
-export async function reconcile(conn, cfg, baseDir) {
-    const files = listFiles(cfg, baseDir);
-    const currentSet = new Set(files.map((f)=>f.relPath));
-    const existingStmt = await conn.prepare('SELECT "path", "_mtime", "_size" FROM frontmatter');
-    const existingRows = await existingStmt.all();
-    const existing = new Map(existingRows.map((r)=>[
-            r.path,
-            r
+async function reconcileContent(conn, touched, docs, _delta) {
+    if (touched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', touched.map((p)=>[
+            p
         ]));
-    const vanished = existingRows.filter((r)=>!currentSet.has(r.path)).map((r)=>r.path);
-    const toReparse = files.filter((f)=>{
-        const row = existing.get(f.relPath);
-        return !row || row._mtime !== f.mtimeMs || row._size !== f.size;
-    });
-    if (vanished.length === 0 && toReparse.length === 0) return {
-        parsed: 0,
-        warnings: []
-    };
-    const features = activeFeatures(cfg);
-    const seenColumns = await getColumns(conn);
-    const report = progress('reparsing files', toReparse.length);
-    const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);
-    report.finish();
-    for (const col of newColumns)seenColumns.add(col);
-    const allColumns = [
-        ...seenColumns
-    ];
-    if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {
-        throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
-    }
-    const writableColumns = allColumns.filter((c)=>CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));
-    const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(()=>'?').join(', ')}) ON CONFLICT("path") DO UPDATE SET ${writableColumns.filter((c)=>c !== 'path').map((c)=>`${quoteIdent(c)} = excluded.${quoteIdent(c)}`).join(', ')}`;
-    const added = toReparse.filter((f)=>!existing.has(f.relPath)).map((f)=>f.relPath);
-    const delta = {
-        files,
-        reparsed: parsedDocs.map((d)=>d.relPath),
-        added,
-        vanished
-    };
-    const addedSet = new Set(added);
-    const reparsedExisting = parsedDocs.map((d)=>d.relPath).filter((p)=>!addedSet.has(p));
-    await withTransaction(conn, async ()=>{
-        var _feature_remove, _feature_afterReconcile;
-        // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
-        // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
-        for (const col of newColumns)await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);
-        if (parsedDocs.length > 0) {
-            const rows = parsedDocs.map((doc)=>writableColumns.map((col)=>{
-                    var _doc_data_col;
-                    if (col === 'path') return doc.relPath;
-                    if (col === '_mtime') return doc.mtimeMs;
-                    if (col === '_ctime') return doc.ctimeMs;
-                    if (col === '_size') return doc.size;
-                    if (col === '_parse_error') return doc.parseError;
-                    return (_doc_data_col = doc.data[col]) !== null && _doc_data_col !== void 0 ? _doc_data_col : null;
-                }));
-            await conn.runBatch(insertSql, rows);
-        }
-        const contentTouched = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (contentTouched.length > 0) await conn.runBatch('DELETE FROM content WHERE "path" = ?', contentTouched.map((p)=>[
-                p
-            ]));
-        if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));
-        if (vanished.length > 0) await conn.runBatch('DELETE FROM frontmatter WHERE "path" = ?', vanished.map((p)=>[
-                p
-            ]));
-        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;
-        // new docs have nothing to clear, which keeps cold builds linear.
-        const presetTouched = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (presetTouched.length > 0) await conn.runBatch('DELETE FROM preset_files WHERE "path" = ?', presetTouched.map((p)=>[
-                p
-            ]));
-        const presetRows = [];
-        for (const doc of parsedDocs)for (const presetName of doc.presets)presetRows.push([
-            doc.relPath,
-            presetName
-        ]);
-        if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files ("path", preset) VALUES (?, ?)', presetRows);
-        const removedPaths = [
-            ...vanished,
-            ...reparsedExisting
-        ];
-        if (removedPaths.length > 0) for (const feature of features)await ((_feature_remove = feature.remove) === null || _feature_remove === void 0 ? void 0 : _feature_remove.call(feature, conn, removedPaths, delta));
-        for (const feature of features){
-            var _feature_store;
-            const docsForFeature = parsedDocs.map((doc)=>({
-                    path: doc.relPath,
-                    extracted: doc.extracted[feature.name]
-                }));
-            await ((_feature_store = feature.store) === null || _feature_store === void 0 ? void 0 : _feature_store.call(feature, conn, docsForFeature, delta));
-        }
-        for (const feature of features)await ((_feature_afterReconcile = feature.afterReconcile) === null || _feature_afterReconcile === void 0 ? void 0 : _feature_afterReconcile.call(feature, conn, delta));
-    });
-    return {
-        parsed: parsedDocs.length,
-        warnings
-    };
+    if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));
 }
+export const duckdbDialect = {
+    beginMode: ()=>'BEGIN',
+    checkColumnLimit (count) {
+        if (count > MAX_FRONTMATTER_COLUMNS) {
+            throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);
+        }
+    },
+    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
+    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
+    columnTypeSuffix: ()=>' VARIANT',
+    reconcileContent
+};