sensemaking 0.20.0 → 0.21.1

package/dist/cjs/store/builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reconcile } from './reconcile.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  build(): Promise<{ parsed: number; warnings: string[] }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// The pool is created at most once, lazily, on whichever build() call first needs it, and reused\n// by every later build() on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: () => reconcile(conn, cfg, baseDir, dialect, pool),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","conn","cfg","baseDir","dialect","pool","ParsePool","build","reconcile","close","poolsCreated"],"mappings":";;;;+BAiBgBA;;;eAAAA;;;sBAhBU;2BACA;AAenB,SAASA,cAAcC,IAAgB,EAAEC,GAAW,EAAEC,OAAe,EAAEC,OAAyB;IACrG,IAAMC,OAAO,IAAIC,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA;mBAAaC,IAAAA,sBAAS,EAACP,MAAMC,KAAKC,SAASC,SAASC;;QACpDI,OAAO,SAAPA;mBAAaJ,KAAKI,KAAK;;QACvB,IAAIC;YACF,OAAOL,KAAKK,YAAY;QAC1B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["createBuilder","nullEmbedVectors","conn","dialect","withTransaction","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","existingPaths","files","docs","paths","delta","prepare","Set","all","map","r","path","listFiles","filter","f","embed","has","relPath","length","reparseFiles","undefined","d","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","feature","docsForFeature","NARROW_FEATURE_TABLE","name","FEATURES","find","Error","turnedOn","afterReconcile","nullRank","columns","getColumns","rerunRank","rank","invalidateFeatureToggles","toggles","byName","linksToggle","rankToggle","Map","get","ParsePool","build","forcedPaths","reconcile","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":";;;;+BA6JgBA;;;eAAAA;;;uBA5JM;uBACG;sBACJ;wBAEK;sBACA;yBACG;8BAGQ;2BACX;wBACC;6BACK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAsBhC,6FAA6F;AAC7F,wEAAwE;AACxE,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;;;;;oBACzE;;wBAAMC,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAAsDF,QAAQG,SAAS;;;oBAAnH;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;;IACrB;;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,SAAeC,kBAAkBN,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe;;YACnHC,cACAC,kBAGAC,OAGEC,MACFC,OACAC;;;;oBATe;;wBAAMf,KAAKgB,OAAO,CAAC;;;oBAAlCN,eAAe;wBACKO;oBAAK;;wBAAMP,aAAaQ,GAAG;;;oBAA/CP,gBAAgB,IAAA,CAAA,EAAA,MAAIM;;wBAAK,cAA0BE,GAAG,CAAC,SAACC;mCAAM,AAACA,EAAuBC,IAAI;;sBAAC;oBACjG,8FAA8F;oBAC9F,4FAA4F;oBACtFT,QAAQU,IAAAA,mBAAS,EAACf,KAAKC,SAASe,MAAM,CAAC,SAACC;+BAAMA,EAAEC,KAAK,IAAId,cAAce,GAAG,CAACF,EAAEG,OAAO;;oBAC1F,IAAIf,MAAMgB,MAAM,KAAK,GAAG;;wBAAO;4BAAEvB,QAAQ;wBAAE;;oBAE1B;;wBAAMwB,IAAAA,uBAAY,EAACjB;4BAAQa,cAAK;2BAAGlB,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAA9EI,OAAS,cAATA;oBACFC,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACjCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,KAAK;wBAAMC,QAAQ;oBAAK;oBAChF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQyB,eACAA;;;;4CADN;;iDAAMA,gBAAAA,cAAK,CAACU,MAAM,cAAZV,oCAAAA,mBAAAA,cAAK,EAAUzB,MAAMc,OAAOC;;;4CAAlC;4CACA;;iDAAMU,eAAAA,cAAK,CAACW,KAAK,cAAXX,mCAAAA,kBAAAA,cAAK,EACTzB,MACAa,KAAKM,GAAG,CAAC,SAACY;2DAAO;wDAAEV,MAAMU,EAAEJ,OAAO;wDAAEU,WAAWN,EAAEM,SAAS,CAACZ,KAAK;oDAAC;oDACjEV;;;4CAHF;;;;;;4BAKF;2BACAd,QAAQG,SAAS;;;oBAVnB;oBAYA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,SAAeU,uBAAuBtC,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE8B,MAAqB;;YAC/IC,OACAC,SAQA7B,OACEC,MACF6B,gBACA5B,OAIAC;;;;oBAhBAyB,QAAQG,oCAAoB,CAACJ,OAAOK,IAAI,CAAC;oBACzCH,UAAUI,iBAAQ,CAACC,IAAI,CAAC,SAACtB;+BAAMA,EAAEoB,IAAI,KAAKL,OAAOK,IAAI;;oBAC3D,IAAI,CAACH,SAAS,MAAM,IAAIM,MAAM,AAAC,wDAAmE,OAAZR,OAAOK,IAAI,EAAC;yBAE9F,CAACL,OAAOS,QAAQ,EAAhB;;;;oBACF;;wBAAM9C,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;2BAAUvC,QAAQG,SAAS;;;oBAAtF;oBACA;;wBAAO;4BAAEC,QAAQ;wBAAE;;;oBAGfO,QAAQU,IAAAA,mBAAS,EAACf,KAAKC;oBACZ;;wBAAMqB,IAAAA,uBAAY,EAACjB;4BAAQ6B;2BAAUlC,KAAK,IAAIU,OAAOa,WAAW;4BAAErB,MAAAA;wBAAK;;;oBAAhFI,OAAS,cAATA;oBACF6B,iBAAiB7B,KAAKM,GAAG,CAAC,SAACY;+BAAO;4BAAEV,MAAMU,EAAEJ,OAAO;4BAAEU,WAAWN,EAAEM,SAAS,CAACI,QAAQG,IAAI,CAAC;wBAAC;;oBAC1F9B,QAAQD,KAAKM,GAAG,CAAC,SAACY;+BAAMA,EAAEJ,OAAO;;oBACvC,gGAAgG;oBAChG,8FAA8F;oBAC9F,8CAA8C;oBACxCZ,QAAwB;wBAAEH,OAAAA;wBAAOoB,UAAUlB;wBAAOmB,OAAOnB;wBAAOoB,QAAQ;oBAAK;oBACnF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCAEQyC,gBACAA;;;;4CAFN;;gDAAMzC,KAAKG,IAAI,CAAC,AAAC,eAAoB,OAANqC;;;4CAA/B;4CACA;;iDAAMC,iBAAAA,QAAQL,KAAK,cAAbK,qCAAAA,oBAAAA,SAAgBzC,MAAM0C,gBAAgB3B;;;4CAA5C;4CACA;;iDAAM0B,0BAAAA,QAAQQ,cAAc,cAAtBR,8CAAAA,6BAAAA,SAAyBzC,MAAMe;;;4CAArC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBAPnB;oBASA;;wBAAO;4BAAEC,QAAQQ,KAAKe,MAAM;wBAAC;;;;IAC/B;;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,SAAesB,SAASlD,IAAgB,EAAEC,OAAyB;;YAC3DkD;;;;oBAAU;;wBAAMC,IAAAA,oBAAU,EAACpD;;;oBAA3BmD,UAAU;oBAChB,IAAI,CAACA,QAAQzB,GAAG,CAAC,UAAU;;;oBAC3B;;wBAAMxB,IAAAA,8BAAe,EAACF,MAAM;mCAAMA,KAAKG,IAAI,CAAC;2BAA0CF,QAAQG,SAAS;;;oBAAvG;;;;;;IACF;;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,SAAeiD,UAAUrD,IAAgB,EAAEC,OAAyB;;YAC5Dc;;;;oBAAAA,QAAwB;wBAAEH,KAAK;wBAAMoB,QAAQ;wBAAMC,KAAK;wBAAMC,QAAQ;oBAAK;oBACjF;;wBAAMhC,IAAAA,8BAAe,EACnBF,MACA;;oCACQsD;;;;4CAAN;;iDAAMA,uBAAAA,YAAI,CAACL,cAAc,cAAnBK,2CAAAA,0BAAAA,YAAI,EAAkBtD,MAAMe;;;4CAAlC;;;;;;4BACF;2BACAd,QAAQG,SAAS;;;oBALnB;;;;;;IAOF;;AAEA,SAAemD,yBAAyBvD,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB,EAAEQ,IAAe,EAAE+C,OAAwB;;YACpJC,QACFpD,QAIEqD,aAWAC,YAMD,2BAAA,mBAAA,gBAAA,WAAA,OAAMpB;;;;oBAtBLkB,SAAS,IAAIG,IAAIJ,QAAQrC,GAAG,CAAC,SAACoB;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL;yBAAO;;oBAChElC,SAAS;oBAEb,4FAA4F;oBAC5F,mFAAmF;oBAC7EqD,cAAcD,OAAOI,GAAG,CAAC;yBAC3BH,aAAAA;;;;oBACS;;wBAAMpB,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAMiD;;;oBAA3ErD,SAAAA,SAAU,AAAC,cAA8EA,MAAM;yBAG3F,CAACqD,YAAYV,QAAQ,EAArB;;;;oBAAuB;;wBAAME,SAASlD,MAAMC;;;oBAArB;;;oBAG7B,wFAAwF;oBACxF,yFAAyF;oBACzF,8EAA8E;oBACxE0D,aAAaF,OAAOI,GAAG,CAAC;yBAC1BF,YAAAA;;;;yBACEA,WAAWX,QAAQ,EAAnBW;;;;oBAAqB;;wBAAMN,UAAUrD,MAAMC;;;oBAAtB;;;;;;oBACpB;;wBAAMiD,SAASlD,MAAMC;;;oBAArB;;;oBAGF,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAgBuD;;;2BAAhB,6BAAA,QAAA;;;;oBAAMjB,SAAN;oBACH,IAAIA,OAAOK,IAAI,KAAK,WAAWL,OAAOK,IAAI,KAAK,QAAQ;;;;oBAC5C;;wBAAMN,uBAAuBtC,MAAMO,KAAKC,SAASP,SAASQ,MAAM8B;;;oBAA3ElC,SAAAA,SAAU,AAAC,cAAyEA,MAAM;;;oBAFvF;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAIL;;wBAAO;4BAAEA,QAAAA;wBAAO;;;;IAClB;;AAIO,SAASP,cAAcE,IAAgB,EAAEO,GAAW,EAAEC,OAAe,EAAEP,OAAyB;IACrG,IAAMQ,OAAO,IAAIqD,iBAAS;IAC1B,OAAO;QACLC,OAAO,SAAPA,MAAQC;mBAAgBC,IAAAA,sBAAS,EAACjE,MAAMO,KAAKC,SAASP,SAASQ,MAAMuD;;QACrEE,YAAY,SAAZA,WAAaC;mBAAUA,SAAS,UAAUpE,iBAAiBC,MAAMC,WAAWK,kBAAkBN,MAAMO,KAAKC,SAASP,SAASQ;;QAC3H2D,oBAAoB,SAApBA,mBAAqBZ;mBAAYD,yBAAyBvD,MAAMO,KAAKC,SAASP,SAASQ,MAAM+C;;QAC7Fa,OAAO,SAAPA;mBAAa5D,KAAK4D,KAAK;;QACvB,IAAIC;YACF,OAAO7D,KAAK6D,YAAY;QAC1B;IACF;AACF"}

package/dist/cjs/store/duckdb/connection.d.cts

@@ -2,4 +2,7 @@ import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection } from '../types.js';
 export declare function storeValueToJs(value: unknown): unknown;
 export declare function storeRowToJs(row: Record<string, unknown>): Record<string, unknown>;
-export declare function createConnection(duckdb: DuckDBConnection): Connection;
+export interface DuckdbConnection extends Connection {
+    readonly duckdb: DuckDBConnection;
+}
+export declare function createConnection(duckdb: DuckDBConnection): DuckdbConnection;

package/dist/cjs/store/duckdb/connection.d.ts

@@ -2,4 +2,7 @@ import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection } from '../types.js';
 export declare function storeValueToJs(value: unknown): unknown;
 export declare function storeRowToJs(row: Record<string, unknown>): Record<string, unknown>;
-export declare function createConnection(duckdb: DuckDBConnection): Connection;
+export interface DuckdbConnection extends Connection {
+    readonly duckdb: DuckDBConnection;
+}
+export declare function createConnection(duckdb: DuckDBConnection): DuckdbConnection;

package/dist/cjs/store/duckdb/connection.js

@@ -568,6 +568,7 @@ var DuckdbStatement = /*#__PURE__*/ function() {
 }();
 function createConnection(duckdb) {
     var conn = {
+        duckdb: duckdb,
         exec: function exec(sql) {
             return _async_to_generator(function() {
                 return _ts_generator(this, function(_state) {

package/dist/cjs/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["createConnection","storeRowToJs","storeValueToJs","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","value","row","out","Object","entries","key","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","register","run","params","result","length","bind","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","duckdb","conn","exec","sql","prepare","runBatch","paramRows","withTransaction","rewritten","stmt","rewriteBatch","flat"],"mappings":";;;;;;;;;;;QAoEgBA;eAAAA;;QAzDAC;eAAAA;;QAHAC;eAAAA;;;6BAPgB;uBAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,gGAAgG;AAChG,yGAAyG;AACzG,IAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AACxC,SAASJ,eAAeK,KAAc;IAC3C,OAAO,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,SAAS,CAACJ,YAAYI,SAASJ,WAAWE,OAAOE,SAASA;AAChG;AACO,SAASN,aAAaO,GAA4B;IACvD,IAAMC,MAA+B,CAAC;QACjC,kCAAA,2BAAA;;QAAL,QAAK,YAAsBC,OAAOC,OAAO,CAACH,yBAArC,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOI,sBAAKL;YAA+BE,GAAG,CAACG,IAAI,GAAGV,eAAeK;;;QAArE;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOE;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,IAAMI,oBAAoB,IAAIC,qBAA8C,SAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,gCAAN;;aAAMA,gBAGQF,QAAiC;gCAHzCE;QAIF,IAAI,CAACF,QAAQ,GAAGA;QAChBF,kBAAkBK,QAAQ,CAAC,IAAI,EAAEH;;iBAL/BE;IAQJ,OAAME,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBC;;;;wBADN,IAAID,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACI,GAAG;;;wBAAhCE,SAAS;wBACf;;4BAAO;gCAAEG,SAASH,OAAOI,WAAW;gCAAEC,iBAAiB;4BAAE;;;;QAC3D;;IAEA,OAAMC,GAGL,GAHD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGP,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACtBQ;;;;wBAAO;;4BAAM,IAAI,CAACC,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAAzBQ,OAAO;wBACb;;4BAAOA,IAAI,CAAC,EAAE;;;;QAChB;;IAEA,OAAMC,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGT,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBU;;;;wBADN,IAAIV,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACgB,aAAa;;;wBAA1CD,SAAS;wBACf;;4BAAQA,OAAOE,eAAe,GAAsCC,GAAG,CAAChC;;;;QAC1E;;IAEA,OAAOiC,OAIN,GAJD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGd,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;;;;wBAG1B;;4BAAA,uBAAM,IAAI,CAACS,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAFzB,4FAA4F;wBAC5F,+DAA+D;wBAC/D;;uCAAA;;oCAAO;;;;;wBAAP;;;;;;QACF;;IAEAe,OAAAA,OAIC,GAJDA,SAAAA;QACE,IAAM1B,MAA+B,EAAE;QACvC,IAAK,IAAI2B,IAAI,GAAGA,IAAI,IAAI,CAACrB,QAAQ,CAACsB,WAAW,EAAED,IAAK3B,IAAI6B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACxB,QAAQ,CAACyB,UAAU,CAACJ;QAAG;QACjG,OAAO3B;IACT;IAEAgC,OAAAA,cAGC,GAHDA,SAAAA,eAAeC,QAAiB;IAC9B,4FAA4F;IAC5F,uFAAuF;IACzF;WAxCIzB;;AA2CC,SAASjB,iBAAiB2C,MAAwB;IACvD,IAAMC,OAAmB;QACjBC,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,OAAOxB,GAAG,CAAC2B;;;4BAAjB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZ7B;4BAAgB;;gCAAM0B,OAAOI,OAAO,CAACD;;;4BAAhD;;gCAAO,IAAA,CAAA,EAAA,MAAI7B;;oCAAgB;kCAAyB;;;;YACtD;;QAGM+B,UAFN,oHAAoH;QACpH,qIAAqI;QACrI,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAU3B,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAM4B,IAAAA,8BAAe,EAACN,MAAM;;4CACpBO,WAEEC,MASFA,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAM5C;;;;oDAbP2C,YAAYE,IAAAA,qBAAY,EAACP,KAAKG,UAAU3B,MAAM;yDAChD6B,WAAAA;;;;oDACW;;wDAAMR,OAAOI,OAAO,CAACI,UAAUL,GAAG;;;oDAAzCM,OAAO;;;;;;;;;oDAEXA,KAAK7B,IAAI,CAAC0B,UAAUK,IAAI;oDACxB;;wDAAMF,KAAKjC,GAAG;;;oDAAd;;;;;;oDAEAiC,KAAKpC,WAAW;;;;;oDAElB;;;;oDAEW;;wDAAM2B,OAAOI,OAAO,CAACD;;;oDAA5BM,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaH;;;2DAAb,6BAAA,QAAA;;;;oDAAMzC,MAAN;oDACH4C,MAAK7B,IAAI,CAACf;oDACV;;wDAAM4C,MAAKjC,GAAG;;;oDAAd;;;oDAFG;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAKLiC,MAAKpC,WAAW;;;;;;;;;;oCAEpB;;;;4BArBA;;;;;;YAsBF;;IACF;IACA,OAAO4B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["createConnection","storeRowToJs","storeValueToJs","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","value","row","out","Object","entries","key","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","register","run","params","result","length","bind","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","duckdb","conn","exec","sql","prepare","runBatch","paramRows","withTransaction","rewritten","stmt","rewriteBatch","flat"],"mappings":";;;;;;;;;;;QA0EgBA;eAAAA;;QA/DAC;eAAAA;;QAHAC;eAAAA;;;6BAPgB;uBAEH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE7B,gGAAgG;AAChG,yGAAyG;AACzG,IAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AACxC,SAASJ,eAAeK,KAAc;IAC3C,OAAO,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAYA,SAAS,CAACJ,YAAYI,SAASJ,WAAWE,OAAOE,SAASA;AAChG;AACO,SAASN,aAAaO,GAA4B;IACvD,IAAMC,MAA+B,CAAC;QACjC,kCAAA,2BAAA;;QAAL,QAAK,YAAsBC,OAAOC,OAAO,CAACH,yBAArC,SAAA,6BAAA,QAAA,yBAAA;YAAA,mCAAA,iBAAOI,sBAAKL;YAA+BE,GAAG,CAACG,IAAI,GAAGV,eAAeK;;;QAArE;QAAA;;;iBAAA,6BAAA;gBAAA;;;gBAAA;sBAAA;;;;IACL,OAAOE;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,IAAMI,oBAAoB,IAAIC,qBAA8C,SAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,gCAAN;;aAAMA,gBAGQF,QAAiC;gCAHzCE;QAIF,IAAI,CAACF,QAAQ,GAAGA;QAChBF,kBAAkBK,QAAQ,CAAC,IAAI,EAAEH;;iBAL/BE;IAQJ,OAAME,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBC;;;;wBADN,IAAID,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACI,GAAG;;;wBAAhCE,SAAS;wBACf;;4BAAO;gCAAEG,SAASH,OAAOI,WAAW;gCAAEC,iBAAiB;4BAAE;;;;QAC3D;;IAEA,OAAMC,GAGL,GAHD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGP,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBACtBQ;;;;wBAAO;;4BAAM,IAAI,CAACC,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAAzBQ,OAAO;wBACb;;4BAAOA,IAAI,CAAC,EAAE;;;;QAChB;;IAEA,OAAMC,GAIL,GAJD,SAAMA;QAAI,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGT,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;gBAEtBU;;;;wBADN,IAAIV,OAAOE,MAAM,GAAG,GAAG,IAAI,CAACP,QAAQ,CAACQ,IAAI,CAACH;wBAC3B;;4BAAM,IAAI,CAACL,QAAQ,CAACgB,aAAa;;;wBAA1CD,SAAS;wBACf;;4BAAQA,OAAOE,eAAe,GAAsCC,GAAG,CAAChC;;;;QAC1E;;IAEA,OAAOiC,OAIN,GAJD,SAAOA;QAAQ,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGd,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;YAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;;;;wBAG1B;;4BAAA,uBAAM,IAAI,CAACS,GAAG,OAAR,IAAI,EAAK,qBAAGT;;;wBAFzB,4FAA4F;wBAC5F,+DAA+D;wBAC/D;;uCAAA;;oCAAO;;;;;wBAAP;;;;;;QACF;;IAEAe,OAAAA,OAIC,GAJDA,SAAAA;QACE,IAAM1B,MAA+B,EAAE;QACvC,IAAK,IAAI2B,IAAI,GAAGA,IAAI,IAAI,CAACrB,QAAQ,CAACsB,WAAW,EAAED,IAAK3B,IAAI6B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACxB,QAAQ,CAACyB,UAAU,CAACJ;QAAG;QACjG,OAAO3B;IACT;IAEAgC,OAAAA,cAGC,GAHDA,SAAAA,eAAeC,QAAiB;IAC9B,4FAA4F;IAC5F,uFAAuF;IACzF;WAxCIzB;;AAiDC,SAASjB,iBAAiB2C,MAAwB;IACvD,IAAMC,OAAyB;QAC7BD,QAAAA;QACME,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMH,OAAOxB,GAAG,CAAC2B;;;4BAAjB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;;;;gCACZ7B;4BAAgB;;gCAAM0B,OAAOI,OAAO,CAACD;;;4BAAhD;;gCAAO,IAAA,CAAA,EAAA,MAAI7B;;oCAAgB;kCAAyB;;;;YACtD;;QAGM+B,UAFN,oHAAoH;QACpH,qIAAqI;QACrI,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD,IAAIA,UAAU3B,MAAM,KAAK,GAAG;;;4BAC5B;;gCAAM4B,IAAAA,8BAAe,EAACN,MAAM;;4CACpBO,WAEEC,MASFA,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAM5C;;;;oDAbP2C,YAAYE,IAAAA,qBAAY,EAACP,KAAKG,UAAU3B,MAAM;yDAChD6B,WAAAA;;;;oDACW;;wDAAMR,OAAOI,OAAO,CAACI,UAAUL,GAAG;;;oDAAzCM,OAAO;;;;;;;;;oDAEXA,KAAK7B,IAAI,CAAC0B,UAAUK,IAAI;oDACxB;;wDAAMF,KAAKjC,GAAG;;;oDAAd;;;;;;oDAEAiC,KAAKpC,WAAW;;;;;oDAElB;;;;oDAEW;;wDAAM2B,OAAOI,OAAO,CAACD;;;oDAA5BM,QAAO;;;;;;;;;oDAEN,kCAAA,2BAAA;;;;;;;;;oDAAA,YAAaH;;;2DAAb,6BAAA,QAAA;;;;oDAAMzC,MAAN;oDACH4C,MAAK7B,IAAI,CAACf;oDACV;;wDAAM4C,MAAKjC,GAAG;;;oDAAd;;;oDAFG;;;;;;;;;;;;oDAAA;oDAAA;;;;;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;;;;;;;;;oDAKLiC,MAAKpC,WAAW;;;;;;;;;;oCAEpB;;;;4BArBA;;;;;;YAsBF;;IACF;IACA,OAAO4B;AACT"}

package/dist/cjs/store/duckdb/open.js

@@ -167,7 +167,7 @@ function _ts_generator(thisArg, body) {
 var DB_FILENAME = 'cache.duckdb';
 var SCHEMA_VERSION = '3';
 // `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.
-// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.
+// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.
 function ensureSchema(_handle, conn, cfg) {
     return _async_to_generator(function() {
         var _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, feature, err;
@@ -431,6 +431,12 @@ var duckdbOpenDialect = {
     reconcileDialect: _reconcilets.duckdbDialect,
     connect: connect,
     close: close,
+    // duckdb's file lock spans the connection's life, and each platform words the refusal its own
+    // way: posix "Could not set lock on file ... Conflicting lock is held in <exe> (PID n)", Windows
+    // "Cannot open file ... being used by another process". Both are the same condition.
+    isLocked: function isLocked(err) {
+        return /Could not set lock on file|being used by another process/.test(err.message);
+    },
     ensureSchema: ensureSchema,
     createStore: function createStore(handle, conn) {
         return (0, _storets.createStore)(handle.instance, handle.duckdb, conn);

package/dist/cjs/store/duckdb/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer; sqlite's configurable `content.tokenize` is not read here.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAUSC;eAAAA;;;uBAnGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,2JAA2J;AAC3J,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACAd,cAAAA;IACAqC,aAAa,SAAbA,YAActB,QAAQb;eAASmC,IAAAA,oBAAW,EAACtB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOmC,IAAAA,uBAAe,EAACnC,KAAKL;;;IAC9B"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/open.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config, ResolvedConfig } from '../../config/index.ts';\nimport { featureSignature } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures, FEATURES } from '../../features/index.ts';\nimport type { OpenResult } from '../open.ts';\nimport { openWithDialect } from '../open.ts';\nimport { getMeta, setMeta } from '../shared.ts';\nimport type { Connection, OpenDialect } from '../types.ts';\nimport { createConnection } from './connection.ts';\nimport { DUCKDB_PACKAGE, duckdbApi } from './native.ts';\nimport { duckdbDialect } from './reconcile.ts';\nimport { registerFunctions } from './sql-functions.ts';\nimport { createStore } from './store.ts';\n\nexport const DB_FILENAME = 'cache.duckdb';\n// Independent of sqlite's SCHEMA_VERSION -- the two stores' cache shapes evolve separately (VARIANT frontmatter columns vs untyped).\n// The store name already joins the feature signature, so switching a config's `store` key rebuilds rather than reusing the other engine's cache.\nexport const SCHEMA_VERSION = '3';\n\nexport type { OpenResult };\n\n// The connection plus this store's own native handles (types.ts's OpenDialect<Handle>): the\n// instance owns the WAL and must outlive the connection borrowed from it.\ninterface DuckdbHandle {\n  instance: DuckDBInstance;\n  duckdb: DuckDBConnection;\n}\n\n// `content` is a plain table (not FTS-virtual): the fts index is built over it lazily, only when a lexical query runs (lexical.ts), and read directly for contains() verification either way.\n// No tokenizer resolution: this store always uses the fts extension's default (porter) stemmer.\nasync function ensureSchema(_handle: DuckdbHandle, conn: Connection, cfg: Config): Promise<void> {\n  await conn.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" DOUBLE, \"_ctime\" DOUBLE, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS content (\"path\" TEXT PRIMARY KEY, title TEXT, summary TEXT, text TEXT)`);\n  await conn.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  await conn.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) {\n    // Native FLOAT[STORE_DIMS] instead of the embed feature's engine-neutral BLOB+scale DDL (vectors.ts). `scale` is kept, unused,\n    // so the feature's shared reconcile-time INSERT/DELETE (features/embed.ts) names a column that exists on both stores.\n    if (feature.name === 'embed') {\n      await conn.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector FLOAT[${STORE_DIMS}], PRIMARY KEY (\"path\", chunk))`);\n      continue;\n    }\n    await feature.schema(conn);\n  }\n  if ((await getMeta(conn, 'schema_version')) === null) await setMeta(conn, 'schema_version', SCHEMA_VERSION);\n  if ((await getMeta(conn, 'features')) === null) await setMeta(conn, 'features', featureSignature(cfg, FEATURES));\n}\n\n// Order matters: the connection must be gone before the instance closes the WAL.\nasync function close(handle: DuckdbHandle): Promise<void> {\n  handle.duckdb.disconnectSync();\n  handle.instance.closeSync();\n}\n\nasync function connect(dbPath: string, _cfg: ResolvedConfig): Promise<{ handle: DuckdbHandle; conn: Connection }> {\n  // Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency, so nothing imports\n  // it as a value until a duckdb tree opens (types-only imports are erased). Installed on first use if missing, shared with sql-functions.ts and vectors.ts via native.ts's duckdbApi.\n  let DuckDBInstance: typeof import('@duckdb/node-api').DuckDBInstance;\n  try {\n    ({ DuckDBInstance } = await duckdbApi());\n  } catch (err) {\n    if (err instanceof SenseError) throw err;\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" needs the ${DUCKDB_PACKAGE} package (${(err as Error).message})`);\n  }\n\n  let duckdb: DuckDBConnection;\n  let instance: DuckDBInstance | undefined;\n  try {\n    // On-disk files default to an older storage format for cross-version compatibility, which rejects VARIANT columns (\"VARIANT\n    // columns are not supported in storage versions prior to v1.5.0\"); this store's dynamic frontmatter columns need VARIANT (reconcile.ts), so the floor is pinned explicitly.\n    instance = await DuckDBInstance.create(dbPath, { storage_compatibility_version: 'v1.5.0' });\n    duckdb = await instance.connect();\n  } catch (err) {\n    // create() may have succeeded before connect() failed: close it, or its WAL stays open.\n    instance?.closeSync();\n    throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" failed to open ${dbPath}: ${(err as Error).message}`);\n  }\n  // A throw below would leak the open instance, whose WAL then locks the .duckdb file undeletable on Windows.\n  try {\n    await registerFunctions(duckdb);\n    const conn = createConnection(duckdb);\n    return { handle: { instance, duckdb }, conn };\n  } catch (err) {\n    await close({ instance, duckdb });\n    throw err;\n  }\n}\n\n// This store's dialect (types.ts's OpenDialect) for the shared orchestration in store/open.ts.\nexport const duckdbOpenDialect: OpenDialect<DuckdbHandle> = {\n  filename: DB_FILENAME,\n  schemaVersion: SCHEMA_VERSION,\n  reconcileDialect: duckdbDialect,\n  connect,\n  close,\n  // duckdb's file lock spans the connection's life, and each platform words the refusal its own\n  // way: posix \"Could not set lock on file ... Conflicting lock is held in <exe> (PID n)\", Windows\n  // \"Cannot open file ... being used by another process\". Both are the same condition.\n  isLocked: (err) => /Could not set lock on file|being used by another process/.test(err.message),\n  ensureSchema,\n  createStore: (handle, conn) => createStore(handle.instance, handle.duckdb, conn),\n};\n\nexport async function openDuckdb(cfg: ResolvedConfig): Promise<OpenResult> {\n  return openWithDialect(cfg, duckdbOpenDialect);\n}\n"],"names":["DB_FILENAME","SCHEMA_VERSION","duckdbOpenDialect","openDuckdb","ensureSchema","_handle","conn","cfg","feature","exec","activeFeatures","name","STORE_DIMS","schema","getMeta","setMeta","featureSignature","FEATURES","close","handle","duckdb","disconnectSync","instance","closeSync","connect","dbPath","_cfg","DuckDBInstance","err","duckdbApi","SenseError","DUCKDB_PACKAGE","message","create","storage_compatibility_version","registerFunctions","createConnection","filename","schemaVersion","reconcileDialect","duckdbDialect","isLocked","test","createStore","openWithDialect"],"mappings":";;;;;;;;;;;QAgBaA;eAAAA;;QAGAC;eAAAA;;QAwEAC;eAAAA;;QAcSC;eAAAA;;;uBAvGW;uBACN;wBACA;wBACc;sBAET;wBACC;4BAEA;wBACS;2BACZ;8BACI;uBACN;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAErB,IAAMH,cAAc;AAGpB,IAAMC,iBAAiB;AAW9B,8LAA8L;AAC9L,gGAAgG;AAChG,SAAeG,aAAaC,OAAqB,EAAEC,IAAgB,EAAEC,GAAW;;YAKzE,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAJX;;wBAAMF,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACA;;wBAAMH,KAAKG,IAAI,CAAC;;;oBAAhB;oBACK,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAiBC,IAAAA,wBAAc,EAACH;;;2BAAhC,6BAAA,QAAA;;;;oBAAMC,UAAN;yBAGCA,CAAAA,QAAQG,IAAI,KAAK,OAAM,GAAvBH;;;;oBACF;;wBAAMF,KAAKG,IAAI,CAAC,AAAC,qIAA+I,OAAXG,mBAAU,EAAC;;;oBAAhK;oBACA;;;;;oBAEF;;wBAAMJ,QAAQK,MAAM,CAACP;;;oBAArB;;;oBAPG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBASA;;wBAAMQ,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAA2C,IAAG,GAA/C;;;;oBAAkD;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,kBAAkBL;;;oBAAtC;;;oBACjD;;wBAAMa,IAAAA,iBAAO,EAACR,MAAM;;;yBAArB,CAAA,AAAC,kBAAqC,IAAG,GAAzC;;;;oBAA4C;;wBAAMS,IAAAA,iBAAO,EAACT,MAAM,YAAYU,IAAAA,yBAAgB,EAACT,KAAKU,kBAAQ;;;oBAA9D;;;;;;;;IAClD;;AAEA,iFAAiF;AACjF,SAAeC,MAAMC,MAAoB;;;YACvCA,OAAOC,MAAM,CAACC,cAAc;YAC5BF,OAAOG,QAAQ,CAACC,SAAS;;;;;IAC3B;;AAEA,SAAeC,QAAQC,MAAc,EAAEC,IAAoB;;YAGrDC,gBAGKC,KAKLR,QACAE,UAMKM,MAQDtB,MAECsB;;;;;;;;;;oBAvBe;;wBAAMC,IAAAA,mBAAS;;;oBAAlCF,iBAAmB,cAAnBA;;;;;;oBACIC;oBACP,IAAIA,AAAG,YAAHA,KAAeE,oBAAU,GAAE,MAAMF;oBACrC,MAAM,IAAIE,oBAAU,CAAC,4BAA4B,AAAC,4BAAsD,OAA3BC,wBAAc,EAAC,cAAmC,OAAvB,AAACH,IAAcI,OAAO,EAAC;;;;;;;;oBAQpH;;wBAAML,eAAeM,MAAM,CAACR,QAAQ;4BAAES,+BAA+B;wBAAS;;;oBAFzF,4HAA4H;oBAC5H,4KAA4K;oBAC5KZ,WAAW;oBACF;;wBAAMA,SAASE,OAAO;;;oBAA/BJ,SAAS;;;;;;oBACFQ;oBACP,wFAAwF;oBACxFN,qBAAAA,+BAAAA,SAAUC,SAAS;oBACnB,MAAM,IAAIO,oBAAU,CAAC,4BAA4B,AAAC,iCAA2C,OAAXL,QAAO,MAA2B,OAAvB,AAACG,KAAcI,OAAO;;;;;;;;oBAInH;;wBAAMG,IAAAA,iCAAiB,EAACf;;;oBAAxB;oBACMd,OAAO8B,IAAAA,8BAAgB,EAAChB;oBAC9B;;wBAAO;4BAAED,QAAQ;gCAAEG,UAAAA;gCAAUF,QAAAA;4BAAO;4BAAGd,MAAAA;wBAAK;;;oBACrCsB;oBACP;;wBAAMV,MAAM;4BAAEI,UAAAA;4BAAUF,QAAAA;wBAAO;;;oBAA/B;oBACA,MAAMQ;;;;;;;IAEV;;AAGO,IAAM1B,oBAA+C;IAC1DmC,UAAUrC;IACVsC,eAAerC;IACfsC,kBAAkBC,0BAAa;IAC/BhB,SAAAA;IACAN,OAAAA;IACA,8FAA8F;IAC9F,iGAAiG;IACjG,qFAAqF;IACrFuB,UAAU,SAAVA,SAAWb;eAAQ,2DAA2Dc,IAAI,CAACd,IAAII,OAAO;;IAC9F5B,cAAAA;IACAuC,aAAa,SAAbA,YAAcxB,QAAQb;eAASqC,IAAAA,oBAAW,EAACxB,OAAOG,QAAQ,EAAEH,OAAOC,MAAM,EAAEd;;AAC7E;AAEO,SAAeH,WAAWI,GAAmB;;;YAClD;;gBAAOqC,IAAAA,uBAAe,EAACrC,KAAKL;;;IAC9B"}

package/dist/cjs/store/duckdb/reconcile.js

@@ -9,7 +9,10 @@ Object.defineProperty(exports, "duckdbDialect", {
     }
 });
 var _errorsts = require("../../errors.js");
+var _reconcilets = require("../reconcile.js");
+var _sharedts = require("../shared.js");
 var _lexicalts = require("./lexical.js");
+var _nativets = require("./native.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -196,6 +199,135 @@ function reconcileContent(conn, touched, docs, _delta) {
         });
     })();
 }
+// DuckDB rejects more than one ALTER command per statement ("Parser Error: Only one ALTER
+// command per statement is supported", measured), so every name's clause joins into one string
+// and runs as a single exec() -- one column-add "leg" through the driver instead of `names.length`.
+// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()
+// produces for one key across files.
+function addColumns(conn, names) {
+    return _async_to_generator(function() {
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (names.length === 0) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        conn.exec(names.map(function(name) {
+                            return "ALTER TABLE frontmatter ADD COLUMN ".concat((0, _sharedts.quoteIdent)(name), " VARIANT");
+                        }).join('; '))
+                    ];
+                case 1:
+                    _state.sent();
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
+// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so
+// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. "_rank") that still exist on the table, and get appendDefault().
+function insertNew(conn, table, columns, rows) {
+    return _async_to_generator(function() {
+        var variantValue, infoStmt, physicalColumns, rowIndexOf, native, appender, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, row, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, name, idx, value;
+        return _ts_generator(this, function(_state) {
+            switch(_state.label){
+                case 0:
+                    if (rows.length === 0) return [
+                        2
+                    ];
+                    return [
+                        4,
+                        (0, _nativets.duckdbApi)()
+                    ];
+                case 1:
+                    variantValue = _state.sent().variantValue;
+                    return [
+                        4,
+                        conn.prepare("PRAGMA table_info(".concat((0, _sharedts.quoteIdent)(table), ")"))
+                    ];
+                case 2:
+                    infoStmt = _state.sent();
+                    return [
+                        4,
+                        infoStmt.all()
+                    ];
+                case 3:
+                    physicalColumns = _state.sent().map(function(c) {
+                        return c.name;
+                    });
+                    rowIndexOf = new Map(columns.map(function(name, i) {
+                        return [
+                            name,
+                            i
+                        ];
+                    }));
+                    native = conn.duckdb;
+                    return [
+                        4,
+                        native.createAppender(table)
+                    ];
+                case 4:
+                    appender = _state.sent();
+                    try {
+                        _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+                        try {
+                            for(_iterator = rows[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+                                row = _step.value;
+                                _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
+                                try {
+                                    for(_iterator1 = physicalColumns[Symbol.iterator](); !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
+                                        name = _step1.value;
+                                        idx = rowIndexOf.get(name);
+                                        value = idx === undefined ? undefined : row[idx];
+                                        if (idx === undefined) appender.appendDefault();
+                                        else if (value === null || value === undefined) appender.appendNull();
+                                        else if (_reconcilets.CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value);
+                                        else appender.appendVariant(variantValue(value));
+                                    }
+                                } catch (err) {
+                                    _didIteratorError1 = true;
+                                    _iteratorError1 = err;
+                                } finally{
+                                    try {
+                                        if (!_iteratorNormalCompletion1 && _iterator1.return != null) {
+                                            _iterator1.return();
+                                        }
+                                    } finally{
+                                        if (_didIteratorError1) {
+                                            throw _iteratorError1;
+                                        }
+                                    }
+                                }
+                                appender.endRow();
+                            }
+                        } catch (err) {
+                            _didIteratorError = true;
+                            _iteratorError = err;
+                        } finally{
+                            try {
+                                if (!_iteratorNormalCompletion && _iterator.return != null) {
+                                    _iterator.return();
+                                }
+                            } finally{
+                                if (_didIteratorError) {
+                                    throw _iteratorError;
+                                }
+                            }
+                        }
+                        appender.flushSync();
+                    } finally{
+                        appender.closeSync();
+                    }
+                    return [
+                        2
+                    ];
+            }
+        });
+    })();
+}
 var duckdbDialect = {
     beginMode: function beginMode() {
         return 'BEGIN';
@@ -205,11 +337,8 @@ var duckdbDialect = {
             throw new _errorsts.SenseError('COLUMN_LIMIT', "frontmatter would need ".concat(count, " columns, crossing this store's sanity limit (").concat(MAX_FRONTMATTER_COLUMNS, "). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys."));
         }
     },
-    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold
-    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.
-    columnTypeSuffix: function columnTypeSuffix() {
-        return ' VARIANT';
-    },
-    reconcileContent: reconcileContent
+    addColumns: addColumns,
+    reconcileContent: reconcileContent,
+    insertNew: insertNew
 };
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n  // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n  columnTypeSuffix: () => ' VARIANT',\n  reconcileContent,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","beginMode","checkColumnLimit","count","SenseError","columnTypeSuffix"],"mappings":";;;;+BAmCaA;;;eAAAA;;;wBAnCc;yBAIM;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAEjC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEO,IAAMX,gBAAkC;IAC7CoB,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQb,yBAAyB;YACnC,MAAM,IAAIc,oBAAU,CAAC,gBAAgB,AAAC,0BAA+Ed,OAAtDa,OAAM,kDAAwE,OAAxBb,yBAAwB;QAC/I;IACF;IACA,yFAAyF;IACzF,2FAA2F;IAC3Fe,kBAAkB,SAAlBA;eAAwB;;IACxBd,kBAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { CORE_FRONTMATTER_COLUMNS } from '../reconcile.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\n// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so\n// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. \"_rank\") that still exist on the table, and get appendDefault().\nasync function insertNew(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const { variantValue } = await duckdbApi();\n  const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n  const physicalColumns = ((await infoStmt.all()) as Array<{ name: string }>).map((c) => c.name);\n  const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n  const native = (conn as DuckdbConnection).duckdb;\n  const appender = await native.createAppender(table);\n  try {\n    for (const row of rows) {\n      for (const name of physicalColumns) {\n        const idx = rowIndexOf.get(name);\n        const value = idx === undefined ? undefined : row[idx];\n        if (idx === undefined) appender.appendDefault();\n        else if (value === null || value === undefined) appender.appendNull();\n        else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value as DuckDBValue);\n        else appender.appendVariant(variantValue(value as DuckDBValue));\n      }\n      appender.endRow();\n    }\n    appender.flushSync();\n  } finally {\n    appender.closeSync();\n  }\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n  insertNew,\n};\n"],"names":["duckdbDialect","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","markContentStale","addColumns","names","exec","name","quoteIdent","join","insertNew","table","columns","rows","variantValue","infoStmt","physicalColumns","rowIndexOf","native","appender","row","idx","value","duckdbApi","prepare","all","c","Map","i","duckdb","createAppender","get","undefined","appendDefault","appendNull","CORE_FRONTMATTER_COLUMNS","has","appendValue","appendVariant","endRow","flushSync","closeSync","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BA+EaA;;;eAAAA;;;wBA9Ec;2BAGc;wBACd;yBAGM;wBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,qBAAqB;AAE3B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAEhC,SAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;;;;;yBACxGF,CAAAA,QAAQG,MAAM,GAAG,CAAA,GAAjBH;;;;oBACF;;wBAAMD,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,SAACC;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;yBAIEL,CAAAA,KAAKE,MAAM,GAAG,CAAA,GAAdF;;;;oBAAiB;;wBAAMF,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;;;oBAAjD;;;oBACrB,6FAA6F;oBAC7F,yCAAyC;oBACzCiB,IAAAA,2BAAgB,EAACR;;;;;;IACnB;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAeS,WAAWT,IAAgB,EAAEU,KAAe;;;;;oBACzD,IAAIA,MAAMN,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMJ,KAAKW,IAAI,CAACD,MAAMJ,GAAG,CAAC,SAACM;mCAAS,AAAC,sCAAsD,OAAjBC,IAAAA,oBAAU,EAACD,OAAM;2BAAWE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEA,mGAAmG;AACnG,uLAAuL;AACvL,SAAeC,UAAUf,IAAgB,EAAEgB,KAAa,EAAEC,OAAiB,EAAEC,IAAiB;;YAEpFC,cACFC,UACAC,iBACAC,YAEAC,QACAC,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACJ,4BAAA,oBAAA,iBAAA,YAAA,QAAMb,MACHc,KACAC;;;;oBAZZ,IAAIT,KAAKd,MAAM,KAAK,GAAG;;;oBACE;;wBAAMwB,IAAAA,mBAAS;;;oBAAhCT,eAAiB,cAAjBA;oBACS;;wBAAMnB,KAAK6B,OAAO,CAAC,AAAC,qBAAsC,OAAlBhB,IAAAA,oBAAU,EAACG,QAAO;;;oBAArEI,WAAW;oBACS;;wBAAMA,SAASU,GAAG;;;oBAAtCT,kBAAkB,AAAE,cAAkDf,GAAG,CAAC,SAACyB;+BAAMA,EAAEnB,IAAI;;oBACvFU,aAAa,IAAIU,IAAIf,QAAQX,GAAG,CAAC,SAACM,MAAMqB;+BAAM;4BAACrB;4BAAMqB;yBAAE;;oBAEvDV,SAAS,AAACvB,KAA0BkC,MAAM;oBAC/B;;wBAAMX,OAAOY,cAAc,CAACnB;;;oBAAvCQ,WAAW;oBACjB,IAAI;wBACG,kCAAA,2BAAA;;4BAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;gCAAbO,MAAN;gCACE,mCAAA,4BAAA;;oCAAL,IAAK,aAAcJ,sCAAd,8BAAA,SAAA,0BAAA,kCAA+B;wCAAzBT,OAAN;wCACGc,MAAMJ,WAAWc,GAAG,CAACxB;wCACrBe,QAAQD,QAAQW,YAAYA,YAAYZ,GAAG,CAACC,IAAI;wCACtD,IAAIA,QAAQW,WAAWb,SAASc,aAAa;6CACxC,IAAIX,UAAU,QAAQA,UAAUU,WAAWb,SAASe,UAAU;6CAC9D,IAAIC,qCAAwB,CAACC,GAAG,CAAC7B,OAAOY,SAASkB,WAAW,CAACf;6CAC7DH,SAASmB,aAAa,CAACxB,aAAaQ;oCAC3C;;oCAPK;oCAAA;;;6CAAA,8BAAA;4CAAA;;;4CAAA;kDAAA;;;;gCAQLH,SAASoB,MAAM;4BACjB;;4BAVK;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBAWLpB,SAASqB,SAAS;oBACpB,SAAU;wBACRrB,SAASsB,SAAS;oBACpB;;;;;;IACF;;AAEO,IAAMzD,gBAAkC;IAC7C0D,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQnD,yBAAyB;YACnC,MAAM,IAAIoD,oBAAU,CAAC,gBAAgB,AAAC,0BAA+EpD,OAAtDmD,OAAM,kDAAwE,OAAxBnD,yBAAwB;QAC/I;IACF;IACAW,YAAAA;IACAV,kBAAAA;IACAgB,WAAAA;AACF"}

package/dist/cjs/store/embed-scope.d.cts

@@ -0,0 +1,2 @@
+export type EmbedChangeKind = 'model' | 'chunk';
+export declare function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null;

package/dist/cjs/store/embed-scope.d.ts

@@ -0,0 +1,2 @@
+export type EmbedChangeKind = 'model' | 'chunk';
+export declare function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null;

package/dist/cjs/store/embed-scope.js

@@ -0,0 +1,85 @@
+// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,
+// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+Object.defineProperty(exports, "classifyEmbedChange", {
+    enumerable: true,
+    get: function() {
+        return classifyEmbedChange;
+    }
+});
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_with_holes(arr) {
+    if (Array.isArray(arr)) return arr;
+}
+function _iterable_to_array_limit(arr, i) {
+    var _i = arr == null ? null : typeof Symbol !== "undefined" && arr[Symbol.iterator] || arr["@@iterator"];
+    if (_i == null) return;
+    var _arr = [];
+    var _n = true;
+    var _d = false;
+    var _s, _e;
+    try {
+        for(_i = _i.call(arr); !(_n = (_s = _i.next()).done); _n = true){
+            _arr.push(_s.value);
+            if (i && _arr.length === i) break;
+        }
+    } catch (err) {
+        _d = true;
+        _e = err;
+    } finally{
+        try {
+            if (!_n && _i["return"] != null) _i["return"]();
+        } finally{
+            if (_d) throw _e;
+        }
+    }
+    return _arr;
+}
+function _non_iterable_rest() {
+    throw new TypeError("Invalid attempt to destructure non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _sliced_to_array(arr, i) {
+    return _array_with_holes(arr) || _iterable_to_array_limit(arr, i) || _unsupported_iterable_to_array(arr, i) || _non_iterable_rest();
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity`. `embed:off`
+// and any segment not in that exact shape return null, so the caller falls back to a full rebuild.
+function parseEmbedSegment(sig) {
+    var part = sig.split('|').find(function(p) {
+        return p.startsWith('embed:');
+    });
+    if (part === undefined || part === 'embed:off') return null;
+    var at = part.indexOf('@');
+    var body = at === -1 ? part : part.slice(0, at);
+    var match = /^embed:(static|openai|cohere):(.+):(chunk:v\d+(?::\d+)?)$/.exec(body);
+    if (!match) return null;
+    var _match = _sliced_to_array(match, 4), provider = _match[1], model = _match[2], chunkVersion = _match[3];
+    return {
+        provider: provider,
+        model: model,
+        chunkVersion: chunkVersion
+    };
+}
+function classifyEmbedChange(before, after) {
+    var b = parseEmbedSegment(before);
+    var a = parseEmbedSegment(after);
+    if (b === null || a === null) return null;
+    if (b.chunkVersion !== a.chunkVersion) return 'chunk';
+    if (b.provider !== a.provider || b.model !== a.model) return 'model';
+    return null;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/store/embed-scope.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n  provider: string;\n  model: string;\n  chunkVersion: string;\n}\n\n// One embed segment's provider/model/chunkVersion, ignoring any trailing `@identity`. `embed:off`\n// and any segment not in that exact shape return null, so the caller falls back to a full rebuild.\nfunction parseEmbedSegment(sig: string): EmbedSegment | null {\n  const part = sig.split('|').find((p) => p.startsWith('embed:'));\n  if (part === undefined || part === 'embed:off') return null;\n  const at = part.indexOf('@');\n  const body = at === -1 ? part : part.slice(0, at);\n  const match = /^embed:(static|openai|cohere):(.+):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (!match) return null;\n  const [, provider, model, chunkVersion] = match;\n  return { provider, model, chunkVersion };\n}\n\n// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);\n// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.\nexport function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null {\n  const b = parseEmbedSegment(before);\n  const a = parseEmbedSegment(after);\n  if (b === null || a === null) return null;\n  if (b.chunkVersion !== a.chunkVersion) return 'chunk';\n  if (b.provider !== a.provider || b.model !== a.model) return 'model';\n  return null;\n}\n"],"names":["classifyEmbedChange","parseEmbedSegment","sig","part","split","find","p","startsWith","undefined","at","indexOf","body","slice","match","exec","provider","model","chunkVersion","before","after","b","a"],"mappings":"AAAA,+FAA+F;AAC/F,oFAAoF;;;;;+BAyBpEA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAfhB,kGAAkG;AAClG,mGAAmG;AACnG,SAASC,kBAAkBC,GAAW;IACpC,IAAMC,OAAOD,IAAIE,KAAK,CAAC,KAAKC,IAAI,CAAC,SAACC;eAAMA,EAAEC,UAAU,CAAC;;IACrD,IAAIJ,SAASK,aAAaL,SAAS,aAAa,OAAO;IACvD,IAAMM,KAAKN,KAAKO,OAAO,CAAC;IACxB,IAAMC,OAAOF,OAAO,CAAC,IAAIN,OAAOA,KAAKS,KAAK,CAAC,GAAGH;IAC9C,IAAMI,QAAQ,4DAA4DC,IAAI,CAACH;IAC/E,IAAI,CAACE,OAAO,OAAO;IACnB,IAA0CA,0BAAAA,WAAjCE,WAAiCF,WAAvBG,QAAuBH,WAAhBI,eAAgBJ;IAC1C,OAAO;QAAEE,UAAAA;QAAUC,OAAAA;QAAOC,cAAAA;IAAa;AACzC;AAIO,SAASjB,oBAAoBkB,MAAc,EAAEC,KAAa;IAC/D,IAAMC,IAAInB,kBAAkBiB;IAC5B,IAAMG,IAAIpB,kBAAkBkB;IAC5B,IAAIC,MAAM,QAAQC,MAAM,MAAM,OAAO;IACrC,IAAID,EAAEH,YAAY,KAAKI,EAAEJ,YAAY,EAAE,OAAO;IAC9C,IAAIG,EAAEL,QAAQ,KAAKM,EAAEN,QAAQ,IAAIK,EAAEJ,KAAK,KAAKK,EAAEL,KAAK,EAAE,OAAO;IAC7D,OAAO;AACT"}

package/dist/cjs/store/feature-scope.d.cts

@@ -0,0 +1,8 @@
+import type { FeatureName } from '../config/index.js';
+export declare const NARROW_FEATURE_TABLE: Readonly<Record<string, string>>;
+export interface FeatureToggle {
+    name: FeatureName;
+    turnedOn: boolean;
+}
+export declare function isFeatureOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function classifyFeatureToggles(before: string, after: string, changedKeys: ReadonlySet<string>): FeatureToggle[] | null;

package/dist/cjs/store/feature-scope.d.ts

@@ -0,0 +1,8 @@
+import type { FeatureName } from '../config/index.js';
+export declare const NARROW_FEATURE_TABLE: Readonly<Record<string, string>>;
+export interface FeatureToggle {
+    name: FeatureName;
+    turnedOn: boolean;
+}
+export declare function isFeatureOnlyChange(changedKeys: ReadonlySet<string>): boolean;
+export declare function classifyFeatureToggles(before: string, after: string, changedKeys: ReadonlySet<string>): FeatureToggle[] | null;

package/dist/cjs/store/feature-scope.js

@@ -0,0 +1,107 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", {
+    value: true
+});
+function _export(target, all) {
+    for(var name in all)Object.defineProperty(target, name, {
+        enumerable: true,
+        get: Object.getOwnPropertyDescriptor(all, name).get
+    });
+}
+_export(exports, {
+    get NARROW_FEATURE_TABLE () {
+        return NARROW_FEATURE_TABLE;
+    },
+    get classifyFeatureToggles () {
+        return classifyFeatureToggles;
+    },
+    get isFeatureOnlyChange () {
+        return isFeatureOnlyChange;
+    }
+});
+function _array_like_to_array(arr, len) {
+    if (len == null || len > arr.length) len = arr.length;
+    for(var i = 0, arr2 = new Array(len); i < len; i++)arr2[i] = arr[i];
+    return arr2;
+}
+function _array_without_holes(arr) {
+    if (Array.isArray(arr)) return _array_like_to_array(arr);
+}
+function _iterable_to_array(iter) {
+    if (typeof Symbol !== "undefined" && iter[Symbol.iterator] != null || iter["@@iterator"] != null) {
+        return Array.from(iter);
+    }
+}
+function _non_iterable_spread() {
+    throw new TypeError("Invalid attempt to spread non-iterable instance.\nIn order to be iterable, non-array objects must have a [Symbol.iterator]() method.");
+}
+function _to_consumable_array(arr) {
+    return _array_without_holes(arr) || _iterable_to_array(arr) || _unsupported_iterable_to_array(arr) || _non_iterable_spread();
+}
+function _unsupported_iterable_to_array(o, minLen) {
+    if (!o) return;
+    if (typeof o === "string") return _array_like_to_array(o, minLen);
+    var n = Object.prototype.toString.call(o).slice(8, -1);
+    if (n === "Object" && o.constructor) n = o.constructor.name;
+    if (n === "Map" || n === "Set") return Array.from(n);
+    if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
+}
+var NARROW_FEATURE_TABLE = {
+    tags: 'tags',
+    sections: 'sections',
+    links: 'links'
+};
+// links feeds rank's afterReconcile pass, so a links toggle always co-changes the `feature:rank`
+// segment too (featureEnabled's dependency); both are narrow-supported and builder.ts applies
+// them together. A changed set naming any other feature falls through to the existing full clear.
+var NARROW_FEATURE_NAMES = new Set(_to_consumable_array(Object.keys(NARROW_FEATURE_TABLE)).concat([
+    'rank'
+]));
+function isFeatureOnlyChange(changedKeys) {
+    return changedKeys.size > 0 && _to_consumable_array(changedKeys).every(function(key) {
+        return key.startsWith('feature:') && NARROW_FEATURE_NAMES.has(key.slice('feature:'.length));
+    });
+}
+// One feature segment's old on/off, read back out of a stored `feature:<name>:on|off` segment.
+// Anything not in that exact shape returns null so the caller falls back to a full rebuild.
+function parseOnOff(sig, name) {
+    var part = sig.split('|').find(function(p) {
+        return p.startsWith("feature:".concat(name, ":"));
+    });
+    if (part === undefined) return null;
+    var fields = part.split(':');
+    if (fields.length !== 3 || fields[2] !== 'on' && fields[2] !== 'off') return null;
+    return fields[2] === 'on';
+}
+function classifyFeatureToggles(before, after, changedKeys) {
+    var toggles = [];
+    var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
+    try {
+        for(var _iterator = changedKeys[Symbol.iterator](), _step; !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true){
+            var key = _step.value;
+            var name = key.slice('feature:'.length);
+            var wasOn = parseOnOff(before, name);
+            var isOn = parseOnOff(after, name);
+            if (wasOn === null || isOn === null || wasOn === isOn) return null;
+            toggles.push({
+                name: name,
+                turnedOn: isOn
+            });
+        }
+    } catch (err) {
+        _didIteratorError = true;
+        _iteratorError = err;
+    } finally{
+        try {
+            if (!_iteratorNormalCompletion && _iterator.return != null) {
+                _iterator.return();
+            }
+        } finally{
+            if (_didIteratorError) {
+                throw _iteratorError;
+            }
+        }
+    }
+    return toggles;
+}
+/* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }