sensemaking 0.19.0 → 0.19.2

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per discovered key, ON CONFLICT upsert). `content` is\n// a plain table, not FTS-virtual, so rows are maintained here unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = await reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["reconcile","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","conn","cfg","baseDir","files","currentSet","existingStmt","existingRows","existing","vanished","toReparse","features","seenColumns","report","parsedDocs","warnings","newColumns","col","allColumns","writableColumns","insertSql","added","delta","addedSet","reparsedExisting","listFiles","map","f","prepare","all","Map","r","path","filter","has","row","get","_mtime","mtimeMs","_size","size","length","parsed","activeFeatures","getColumns","progress","reparseFiles","tick","docs","finish","add","SenseError","c","RESERVED_COLUMNS","quoteIdent","join","reparsed","d","p","withTransaction","feature","rows","contentTouched","presetTouched","presetRows","presetName","removedPaths","exec","ctimeMs","parseError","data","runBatch","presets","push","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":";;;;+BA4BsBA;;;eAAAA;;;wBA3BK;uBACI;0BAEN;wBAEmB;yBACf;wBACU;6BACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGhC,oIAAoI;AACpI,wOAAwO;AACxO,IAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEzB,SAAeX,UAAUY,IAAgB,EAAEC,GAAW,EAAEC,OAAe;;YACtEC,OACAC,YAEAC,cACAC,cACAC,UACAC,UAEAC,WAOAC,UACAC,aAEAC,QAC6C,MAArCC,YAAYC,UAAUC,YAE/B,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAELC,YAIAC,iBACAC,WAKAC,OACAC,OACAC,UACAC;;;;oBApCApB,QAAQqB,IAAAA,mBAAS,EAACvB,KAAKC;oBACvBE,aAAa,IAAId,IAAIa,MAAMsB,GAAG,CAAC,SAACC;+BAAMA,EAAEhC,OAAO;;oBAEhC;;wBAAMM,KAAK2B,OAAO,CAAC;;;oBAAlCtB,eAAe;oBACC;;wBAAMA,aAAauB,GAAG;;;oBAAtCtB,eAAgB;oBAChBC,WAAW,IAAIsB,IAAIvB,aAAamB,GAAG,CAAC,SAACK;+BAAM;4BAACA,EAAEC,IAAI;4BAAED;yBAAE;;oBACtDtB,WAAWF,aAAa0B,MAAM,CAAC,SAACF;+BAAM,CAAC1B,WAAW6B,GAAG,CAACH,EAAEC,IAAI;uBAAGN,GAAG,CAAC,SAACK;+BAAMA,EAAEC,IAAI;;oBAEhFtB,YAAYN,MAAM6B,MAAM,CAAC,SAACN;wBAC9B,IAAMQ,MAAM3B,SAAS4B,GAAG,CAACT,EAAEhC,OAAO;wBAClC,OAAO,CAACwC,OAAOA,IAAIE,MAAM,KAAKV,EAAEW,OAAO,IAAIH,IAAII,KAAK,KAAKZ,EAAEa,IAAI;oBACjE;oBAEA,IAAI/B,SAASgC,MAAM,KAAK,KAAK/B,UAAU+B,MAAM,KAAK,GAAG;;wBAAO;4BAAEC,QAAQ;4BAAG3B,QAAQ;wBAAK;;oBAEhFJ,WAAWgC,IAAAA,uBAAc,EAACzC;oBACZ;;wBAAM0C,IAAAA,oBAAU,EAAC3C;;;oBAA/BW,cAAc;oBAEdC,SAASgC,IAAAA,oBAAQ,EAAC,mBAAmBnC,UAAU+B,MAAM;oBACR;;wBAAMK,IAAAA,uBAAY,EAACpC,WAAWC,UAAUT,KAAKU,aAAaC,OAAOkC,IAAI;;;oBAArE,OAAA,eAArCjC,aAAqC,KAA3CkC,MAAkBjC,WAAyB,KAAzBA,UAAUC,aAAe,KAAfA;oBACpCH,OAAOoC,MAAM;oBACR,kCAAA,2BAAA;;wBAAL,IAAK,YAAajC,iCAAb,6BAAA,QAAA,yBAAA;4BAAMC,MAAN;4BAAyBL,YAAYsC,GAAG,CAACjC;;;wBAAzC;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAECC,aAAc,qBAAGN;oBACvB,IAAIM,WAAWuB,MAAM,GAAGzC,yBAAyB;wBAC/C,MAAM,IAAImD,oBAAU,CAAC,gBAAgB,AAAC,0BAA2FnD,OAAlEkB,WAAWuB,MAAM,EAAC,kDAAwE,OAAxBzC,yBAAwB;oBAC3J;oBACMmB,kBAAkBD,WAAWe,MAAM,CAAC,SAACmB;+BAAM9D,yBAAyB4C,GAAG,CAACkB,MAAM,CAACC,0BAAgB,CAACnB,GAAG,CAACkB;;oBACpGhC,YAAY,AAAC,4BAAkFD,OAAvDA,gBAAgBO,GAAG,CAAC4B,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAA4FpC,OAAhFA,gBAAgBO,GAAG,CAAC;+BAAM;uBAAK6B,IAAI,CAAC,OAAM,wCAGjI,OAHuKpC,gBAClLc,MAAM,CAAC,SAACmB;+BAAMA,MAAM;uBACpB1B,GAAG,CAAC,SAAC0B;+BAAM,AAAC,GAA8BE,OAA5BA,IAAAA,oBAAU,EAACF,IAAG,gBAA4B,OAAdE,IAAAA,oBAAU,EAACF;uBACrDG,IAAI,CAAC;oBAEFlC,QAAQX,UAAUuB,MAAM,CAAC,SAACN;+BAAM,CAACnB,SAAS0B,GAAG,CAACP,EAAEhC,OAAO;uBAAG+B,GAAG,CAAC,SAACC;+BAAMA,EAAEhC,OAAO;;oBAC9E2B,QAAwB;wBAAElB,OAAAA;wBAAOoD,UAAU1C,WAAWY,GAAG,CAAC,SAAC+B;mCAAMA,EAAE9D,OAAO;;wBAAG0B,OAAAA;wBAAOZ,UAAAA;oBAAS;oBAC7Fc,WAAW,IAAIhC,IAAI8B;oBACnBG,mBAAmBV,WAAWY,GAAG,CAAC,SAAC+B;+BAAMA,EAAE9D,OAAO;uBAAEsC,MAAM,CAAC,SAACyB;+BAAM,CAACnC,SAASW,GAAG,CAACwB;;oBAEtF;;wBAAMC,IAAAA,8BAAe,EAAC1D,MAAM;;oCA8CyC2D,iBAK7BA,yBAhDjC,2BAAA,mBAAA,gBAAA,WAAA,OAAM3C,UAGH4C,MAaFC,gBAgBAC,eAMAC,YAC6B,4BAAA,oBAAA,iBAA9B,4BAAA,oBAAA,iBAAA,YAAA,QAAMtE,KAAwB,YAAA,QAAMuE,YAGnCC,cAC4B,4BAAA,oBAAA,iBAAA,YAAA,QAAMN,cACnC,4BAAA,oBAAA,wBAAA,YAAA,aAIA,4BAAA,oBAAA,iBAAA,YAAA,QAAMA;;;;4CAhDN,kCAAA,2BAAA;;;;;;;;;4CAAA,YAAa5C;;;mDAAb,6BAAA,QAAA;;;;4CAAMC,MAAN;4CAAyB;;gDAAMhB,KAAKkE,IAAI,CAAC,AAAC,sCAAqD,OAAhBb,IAAAA,oBAAU,EAACrC,MAAK;;;4CAAtE;;;4CAAzB;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;iDAEDH,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CACI+C,OAAO/C,WAAWY,GAAG,CAAC,SAAChC;uDAC3ByB,gBAAgBO,GAAG,CAAC,SAACT;wDAMZvB;oDALP,IAAIuB,QAAQ,QAAQ,OAAOvB,IAAIC,OAAO;oDACtC,IAAIsB,QAAQ,UAAU,OAAOvB,IAAI4C,OAAO;oDACxC,IAAIrB,QAAQ,UAAU,OAAOvB,IAAI0E,OAAO;oDACxC,IAAInD,QAAQ,SAAS,OAAOvB,IAAI8C,IAAI;oDACpC,IAAIvB,QAAQ,gBAAgB,OAAOvB,IAAI2E,UAAU;oDACjD,QAAO3E,gBAAAA,IAAI4E,IAAI,CAACrD,IAAI,cAAbvB,2BAAAA,gBAAiB;gDAC1B;;4CAEF;;gDAAMO,KAAKsE,QAAQ,CAACnD,WAAWyC;;;4CAA/B;;;4CAGIC,iBAAiB,AAAC,qBAAGrD,iBAAU,qBAAGe;iDACpCsC,CAAAA,eAAerB,MAAM,GAAG,CAAA,GAAxBqB;;;;4CACF;;gDAAM7D,KAAKsE,QAAQ,CACjB,wCACAT,eAAepC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF/B;;;iDAIE5C,CAAAA,WAAW2B,MAAM,GAAG,CAAA,GAApB3B;;;;4CAAuB;;gDAAMb,KAAKsE,QAAQ,CAAC/E,oBAAoBsB,WAAWY,GAAG,CAACjC;;;4CAAvD;;;iDAEvBgB,CAAAA,SAASgC,MAAM,GAAG,CAAA,GAAlBhC;;;;4CACF;;gDAAMR,KAAKsE,QAAQ,CACjB,4CACA9D,SAASiB,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAFzB;;;4CAKF,0FAA0F;4CAC1F,kEAAkE;4CAC5DK,gBAAgB,AAAC,qBAAGtD,iBAAU,qBAAGe;iDACnCuC,CAAAA,cAActB,MAAM,GAAG,CAAA,GAAvBsB;;;;4CACF;;gDAAM9D,KAAKsE,QAAQ,CACjB,6CACAR,cAAcrC,GAAG,CAAC,SAACgC;2DAAM;wDAACA;qDAAE;;;;4CAF9B;;;4CAIIM;4CAC6B,mCAAA,4BAAA,6BAA9B,mCAAA,4BAAA;;gDAAL,IAAK,aAAalD,iCAAb,8BAAA,SAAA,0BAAA;oDAAMpB,MAAN;;wDAAyB,IAAK,aAAoBA,IAAI8E,OAAO,uBAA/B,8BAAA,SAAA,0BAAA;4DAAMP,aAAN;4DAAiCD,WAAWS,IAAI;gEAAE/E,IAAIC,OAAO;gEAAEsE;;;;wDAA/D;wDAAA;;;iEAAA,8BAAA;gEAAA;;;gEAAA;sEAAA;;;;;;gDAA9B;gDAAA;;;yDAAA,8BAAA;wDAAA;;;wDAAA;8DAAA;;;;iDACDD,CAAAA,WAAWvB,MAAM,GAAG,CAAA,GAApBuB;;;;4CAAuB;;gDAAM/D,KAAKsE,QAAQ,CAAC,2DAA2DP;;;4CAA/E;;;4CAErBE,eAAe,AAAC,qBAAGzD,iBAAU,qBAAGe;4CACJ,mCAAA,4BAAA;iDAA9B0C,CAAAA,aAAazB,MAAM,GAAG,CAAA,GAAtByB;;;;;;;;;;;;4CAA8B,aAAiBvD;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,UAAN;4CAA2B;;iDAAMA,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiB3D,MAAMiE,cAAc5C;;;4CAA3C;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAC7B,mCAAA,4BAAA;;;;;;;;;;oDAAMsC,SAEHA,gBADAe;;;;4DADGf,UAAN;4DACGe,iBAAiC7D,WAAWY,GAAG,CAAC,SAAChC;uEAAS;oEAAEsC,MAAMtC,IAAIC,OAAO;oEAAEiF,WAAWlF,IAAIkF,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gEAAC;;4DAC5H;;iEAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgB3D,MAAM0E,gBAAgBrD;;;4DAA5C;;;;;;4CACF;4CAHK,aAAiBX;;;mDAAjB,8BAAA,SAAA;;;;;;;;;;;;4CAAA;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;4CAIA,mCAAA,4BAAA;;;;;;;;;4CAAA,aAAiBA;;;mDAAjB,8BAAA,SAAA;;;;4CAAMiD,WAAN;4CAA2B;;iDAAMA,0BAAAA,SAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,UAAyB3D,MAAMqB;;;4CAArC;;;4CAA3B;;;;;;;;;;;;4CAAA;4CAAA;;;;;;;qDAAA,8BAAA;oDAAA;;;oDAAA;0DAAA;;;;;;;;;;;;4BACP;;;;oBApDA;oBAsDA;;wBAAO;4BAAEoB,QAAQ5B,WAAW2B,MAAM;4BAAE1B,UAAAA;wBAAS;;;;IAC/C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","beginMode","checkColumnLimit","count","SenseError","columnTypeSuffix"],"mappings":";;;;+BA+BaA;;;eAAAA;;;wBA/Bc;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK3B,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;;;;;;IACvB;;AAEO,IAAMF,gBAAkC;IAC7CmB,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQZ,yBAAyB;YACnC,MAAM,IAAIa,oBAAU,CAAC,gBAAgB,AAAC,0BAA+Eb,OAAtDY,OAAM,kDAAwE,OAAxBZ,yBAAwB;QAC/I;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fc,kBAAkB,SAAlBA;eAAwB;;IACxBb,kBAAAA;AACF"}

package/dist/cjs/store/duckdb/store.js

@@ -17,13 +17,14 @@ _export(exports, {
     }
 });
 var _typests = require("../../embed/types.js");
+var _reconcilets = require("../reconcile.js");
 var _sharedts = require("../shared.js");
 var _transactionts = require("../transaction.js");
 var _vectorsts = require("../vectors.js");
 var _connectionts = require("./connection.js");
 var _fieldStatsts = require("./fieldStats.js");
 var _lexicalts = require("./lexical.js");
-var _reconcilets = require("./reconcile.js");
+var _reconcilets1 = require("./reconcile.js");
 var _vectorsts1 = require("./vectors.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -266,7 +267,8 @@ function _wrap_async_generator(fn) {
 var CAPABILITIES = new Set([
     'lexical',
     'phrases',
-    'vectors'
+    'vectors',
+    'sql-functions'
 ]);
 function createStore(instance, duckdb, conn, cfg, baseDir) {
     var lex = (0, _lexicalts.createLexicalIndex)(conn);
@@ -337,7 +339,7 @@ function createStore(instance, duckdb, conn, cfg, baseDir) {
                         case 0:
                             return [
                                 4,
-                                (0, _reconcilets.reconcile)(conn, cfg, baseDir)
+                                (0, _reconcilets.reconcile)(conn, cfg, baseDir, _reconcilets1.duckdbDialect)
                             ];
                         case 1:
                             result = _state.sent();

package/dist/cjs/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","cfg","baseDir","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","result","markStale","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAeaA;eAAAA;;QAIGC;eAAAA;;;uBAjBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;2BACT;0BACoC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;CAAU;AAIvF,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,IAAMC,MAAMC,IAAAA,6BAAkB,EAACJ;IAC/B,OAAO;QACLK,MAAM;QACNC,cAAcX;QACRY,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMR,KAAKO,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOR,KAAKS,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMX,KAAKU,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACd,MAAMa;;;YAC/B;;QACME,WAAN,SAAMA;;oBACEC;;;;4BAAS;;gCAAMD,IAAAA,sBAAS,EAACf,MAAMC,KAAKC;;;4BAApCc,SAAS;4BACf,uFAAuF;4BACvF,4DAA4D;4BAC5Db,IAAIc,SAAS;4BACb;;gCAAOD;;;;YACT;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACpB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAqB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACrB,MAAMmB,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOrB,IAAIqB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAAC3B;;YAC3B4B,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC/B,QAAQC,MAAM+B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACtC,QAAQkC,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAAC1C,QAAQC,MAAM+B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAAC3C,MAAMuC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGpC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBsC;;;;gCAAW;;oCAAM/C,OAAOU,OAAO,CAACD;;;gCAAhCsC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE7C,MAAMyC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjFjE,OAAOkE,cAAc;oBACrBnE,SAASoE,SAAS;;;;;YACpB;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { reconcile } from '../reconcile.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. 'snippets' is declined (JS excerpt fallback handles it); 'phrases' means quoted-phrase only -- FTS5 operator syntax is rejected, not answered differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors', 'sql-functions']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir, duckdbDialect);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","cfg","baseDir","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","reconcile","result","duckdbDialect","markStale","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAIGC;eAAAA;;;uBAlBW;2BACD;wBACC;6BACK;yBAEU;4BACb;4BACF;yBACQ;4BACL;0BACgC;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;IAAW;IAAW;CAAgB;AAIxG,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,IAAMC,MAAMC,IAAAA,6BAAkB,EAACJ;IAC/B,OAAO;QACLK,MAAM;QACNC,cAAcX;QACRY,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMR,KAAKO,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAOR,KAAKS,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMX,KAAKU,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACd,MAAMa;;;YAC/B;;QACME,WAAN,SAAMA;;oBACEC;;;;4BAAS;;gCAAMD,IAAAA,sBAAS,EAACf,MAAMC,KAAKC,SAASe,2BAAa;;;4BAA1DD,SAAS;4BACf,uFAAuF;4BACvF,4DAA4D;4BAC5Db,IAAIe,SAAS;4BACb;;gCAAOF;;;;YACT;;QACAG,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACrB;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAsB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAACtB,MAAMoB,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOtB,IAAIsB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAAC5B;;YAC3B6B,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAChC,QAAQC,MAAMgC,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACvC,QAAQmC,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAAC3C,QAAQC,MAAMgC,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAAC5C,MAAMwC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGrC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBuC;;;;gCAAW;;oCAAMhD,OAAOU,OAAO,CAACD;;;gCAAhCuC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE9C,MAAM0C,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjFlE,OAAOmE,cAAc;oBACrBpE,SAASqE,SAAS;;;;;YACpB;;IACF;AACF"}

package/dist/cjs/store/index.d.cts

@@ -7,3 +7,4 @@ export { getMeta, setMeta } from './shared.js';
 export type { OpenResult } from './sqlite/open.js';
 export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
+export { CAPABILITY_NAMES } from './types.js';

package/dist/cjs/store/index.d.ts

@@ -7,3 +7,4 @@ export { getMeta, setMeta } from './shared.js';
 export type { OpenResult } from './sqlite/open.js';
 export { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
+export { CAPABILITY_NAMES } from './types.js';

package/dist/cjs/store/index.js

@@ -11,6 +11,9 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get CAPABILITY_NAMES () {
+        return _typests.CAPABILITY_NAMES;
+    },
     get DB_FILENAME () {
         return _opents1.DB_FILENAME;
     },
@@ -46,6 +49,7 @@ var _opents2 = require("./turso/open.js");
 var _storets2 = require("./turso/store.js");
 var _cachets = require("./cache.js");
 var _sharedts = require("./shared.js");
+var _typests = require("./types.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);

package/dist/cjs/store/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","open","openSqlite","duckdb","DUCKDB_CAPABILITIES","openDuckdb","turso","TURSO_CAPABILITIES","openTurso","entryFor","name","entry","SenseError","Object","keys","join","cfg","storeName","anyPresetEmbeds","has"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;;;;;;;;;;;;QA0D9CA;eAAAA,oBAAW;;QAAYC;eAAAA,uBAAc;;QAHrCC;eAAAA,mBAAU;;QAGGC;eAAAA,iBAAQ;;QAFrBC;eAAAA,iBAAO;;QAvBMC;eAAAA;;QAeNC;eAAAA;;QAQEC;eAAAA,iBAAO;;;uBArDkB;wBAChB;sBACA;uBACyB;uBAEzB;wBACyB;uBAC1B;wBACyB;uBA4CxB;wBACM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AArCjC,kGAAkG;AAClG,4EAA4E;AAC5E,IAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcC,sBAAmB;QAAEC,MAAMC,mBAAU;IAAC;IAC9DC,QAAQ;QAAEJ,cAAcK,qBAAmB;QAAEH,MAAMI,kBAAU;IAAC;IAC9DC,OAAO;QAAEP,cAAcQ,sBAAkB;QAAEN,MAAMO,kBAAS;IAAC;AAC7D;AAEA,SAASC,SAASC,IAAe;IAC/B,IAAMC,QAAQd,QAAQ,CAACa,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIC,oBAAU,CAAC,iBAAiB,AAAC,0BAA8CC,OAArBH,MAAK,kBAAiD,OAAjCG,OAAOC,IAAI,CAACjB,UAAUkB,IAAI,CAAC;IAC5H,OAAOJ;AACT;AAEO,SAAejB,UAAUsB,GAAmB;;YAC3CN,MACAC;;YADAD,OAAOO,IAAAA,kBAAS,EAACD;YACjBL,QAAQF,SAASC;YAEvB,8FAA8F;YAC9F,oJAAoJ;YACpJ,IAAIQ,IAAAA,wBAAe,EAACF,QAAQ,CAACL,MAAMZ,YAAY,CAACoB,GAAG,CAAC,YAAY;gBAC9D,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;YAClE;YAEA;;gBAAOC,MAAMV,IAAI,CAACe;;;IACpB;;AAIO,SAASrB,wBAAwBqB,GAAmB;IACzD,IAAMN,OAAOO,IAAAA,kBAAS,EAACD;IACvB,IAAI,CAACP,SAASC,MAAMX,YAAY,CAACoB,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;IAClE;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig, StoreName } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport { openTurso } from './turso/open.ts';\nimport { CAPABILITIES as TURSO_CAPABILITIES } from './turso/store.ts';\nimport type { Capability } from './types.ts';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n  turso: { capabilities: TURSO_CAPABILITIES, open: openTurso },\n};\n\nfunction entryFor(name: StoreName): StoreEntry {\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n  return entry;\n}\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = entryFor(name);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually uses\n  // for vectors. Lexical search has no such signal at this level (every preset defaults to \"words\"), so a store lacking it fails loudly at first use.\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\n// watch keeps its connection open for the run, so a file-locking store would fail every\n// other command on the tree; check before open, as openStore does for \"vectors\".\nexport function requireWatchConcurrency(cfg: ResolvedConfig): void {\n  const name = storeName(cfg);\n  if (!entryFor(name).capabilities.has('watch-concurrency')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"watch-concurrency\" in this build; sense watch holds the store open for its whole run, and this store locks the cache file, so every other command on this tree would fail to open; set \"store\" to \"sqlite\" in sense.config.json`);\n  }\n}\n\nexport { clearCache } from './cache.ts';\nexport { getMeta, setMeta } from './shared.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\nexport { CAPABILITY_NAMES } from './types.ts';\n"],"names":["CAPABILITY_NAMES","DB_FILENAME","SCHEMA_VERSION","clearCache","docCount","getMeta","openStore","requireWatchConcurrency","setMeta","REGISTRY","sqlite","capabilities","SQLITE_CAPABILITIES","open","openSqlite","duckdb","DUCKDB_CAPABILITIES","openDuckdb","turso","TURSO_CAPABILITIES","openTurso","entryFor","name","entry","SenseError","Object","keys","join","cfg","storeName","anyPresetEmbeds","has"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;;;;;;;;;;;;QA4D9CA;eAAAA,yBAAgB;;QAFhBC;eAAAA,oBAAW;;QAAYC;eAAAA,uBAAc;;QAHrCC;eAAAA,mBAAU;;QAGGC;eAAAA,iBAAQ;;QAFrBC;eAAAA,iBAAO;;QAvBMC;eAAAA;;QAeNC;eAAAA;;QAQEC;eAAAA,iBAAO;;;uBArDkB;wBAChB;sBACA;uBACyB;uBAEzB;wBACyB;uBAC1B;wBACyB;uBA4CxB;wBACM;uBAIA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAzCjC,kGAAkG;AAClG,4EAA4E;AAC5E,IAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcC,sBAAmB;QAAEC,MAAMC,mBAAU;IAAC;IAC9DC,QAAQ;QAAEJ,cAAcK,qBAAmB;QAAEH,MAAMI,kBAAU;IAAC;IAC9DC,OAAO;QAAEP,cAAcQ,sBAAkB;QAAEN,MAAMO,kBAAS;IAAC;AAC7D;AAEA,SAASC,SAASC,IAAe;IAC/B,IAAMC,QAAQd,QAAQ,CAACa,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAIC,oBAAU,CAAC,iBAAiB,AAAC,0BAA8CC,OAArBH,MAAK,kBAAiD,OAAjCG,OAAOC,IAAI,CAACjB,UAAUkB,IAAI,CAAC;IAC5H,OAAOJ;AACT;AAEO,SAAejB,UAAUsB,GAAmB;;YAC3CN,MACAC;;YADAD,OAAOO,IAAAA,kBAAS,EAACD;YACjBL,QAAQF,SAASC;YAEvB,8FAA8F;YAC9F,oJAAoJ;YACpJ,IAAIQ,IAAAA,wBAAe,EAACF,QAAQ,CAACL,MAAMZ,YAAY,CAACoB,GAAG,CAAC,YAAY;gBAC9D,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;YAClE;YAEA;;gBAAOC,MAAMV,IAAI,CAACe;;;IACpB;;AAIO,SAASrB,wBAAwBqB,GAAmB;IACzD,IAAMN,OAAOO,IAAAA,kBAAS,EAACD;IACvB,IAAI,CAACP,SAASC,MAAMX,YAAY,CAACoB,GAAG,CAAC,sBAAsB;QACzD,MAAM,IAAIP,oBAAU,CAAC,4BAA4B,AAAC,UAAc,OAALF,MAAK;IAClE;AACF"}

package/dist/cjs/store/reconcile.d.cts

@@ -0,0 +1,6 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;

package/dist/cjs/store/reconcile.d.ts

@@ -0,0 +1,6 @@
+import type { Config } from '../config/index.js';
+import type { Connection, ReconcileDialect } from './types.js';
+export declare function reconcile(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Promise<{
+    parsed: number;
+    warnings: string[];
+}>;