sensemaking 0.17.0 → 0.18.0

package/dist/esm/store/duckdb/reconcile.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { Config } from '../../config/index.ts';\nimport { SenseError } from '../../errors.ts';\nimport { activeFeatures } from '../../features/index.ts';\nimport type { ExtractedDoc, ReconcileDelta } from '../../features/types.ts';\nimport { progress } from '../../output/progress.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { listFiles, RESERVED_COLUMNS } from '../../scan/index.ts';\nimport { reparseFiles } from '../../scan/reparse.ts';\nimport { getColumns, quoteIdent } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection } from '../types.ts';\n\n// Mirrors src/store/sqlite/reconcile.ts's frontmatter-upsert shape (dynamic ALTER TABLE per\n// discovered key, ON CONFLICT upsert keeping the row's identity stable). `content` is a plain\n// table, not FTS-virtual (D1: DuckDB's fts index is built lazily, on first lexical query, from\n// this table -- see duckdb/lexical.ts), so its rows are maintained here unconditionally. The one\n// forced divergence for frontmatter itself is the ALTER TABLE type: DuckDB requires a declared\n// column type where sqlite accepts none, so every dynamic frontmatter column is VARIANT (the\n// only DuckDB type that can hold the different JS types mapValue() produces across files for\n// the same key).\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity\n// fence anyway so a runaway frontmatter generator fails with a clear message instead of an\n// unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nexport async function reconcile(conn: Connection, cfg: Config, baseDir: string): Promise<{ parsed: number; warnings: string[] }> {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingStmt = await conn.prepare('SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter');\n  const existingRows = (await existingStmt.all()) as Array<{ path: string; _mtime: number; _size: number }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = await getColumns(conn);\n\n  const report = progress('reparsing files', toReparse.length);\n  const { docs: parsedDocs, warnings, newColumns } = reparseFiles(toReparse, features, cfg, seenColumns, report.tick);\n  report.finish();\n  for (const col of newColumns) seenColumns.add(col);\n\n  const allColumns = [...seenColumns];\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${allColumns.length} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n  }\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n  const addedSet = new Set(added);\n  const reparsedExisting = parsedDocs.map((d) => d.relPath).filter((p) => !addedSet.has(p));\n\n  await withTransaction(conn, async () => {\n    // DuckDB's ALTER TABLE ADD COLUMN requires a type; VARIANT is the only one that can hold\n    // the mixed bigint/number/string/null shapes mapValue() produces for one key across files.\n    for (const col of newColumns) await conn.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)} VARIANT`);\n\n    if (parsedDocs.length > 0) {\n      const rows = parsedDocs.map((doc) =>\n        writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        })\n      );\n      await conn.runBatch(insertSql, rows);\n    }\n\n    const contentTouched = [...vanished, ...reparsedExisting];\n    if (contentTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM content WHERE \"path\" = ?',\n        contentTouched.map((p) => [p])\n      );\n    if (parsedDocs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, parsedDocs.map(contentRow));\n\n    if (vanished.length > 0)\n      await conn.runBatch(\n        'DELETE FROM frontmatter WHERE \"path\" = ?',\n        vanished.map((p) => [p])\n      );\n\n    // A preset edit forces a full rebuild, so an unchanged doc's coverage is already correct;\n    // new docs have nothing to clear, which keeps cold builds linear.\n    const presetTouched = [...vanished, ...reparsedExisting];\n    if (presetTouched.length > 0)\n      await conn.runBatch(\n        'DELETE FROM preset_files WHERE \"path\" = ?',\n        presetTouched.map((p) => [p])\n      );\n    const presetRows: unknown[][] = [];\n    for (const doc of parsedDocs) for (const presetName of doc.presets) presetRows.push([doc.relPath, presetName]);\n    if (presetRows.length > 0) await conn.runBatch('INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)', presetRows);\n\n    const removedPaths = [...vanished, ...reparsedExisting];\n    if (removedPaths.length > 0) for (const feature of features) await feature.remove?.(conn, removedPaths, delta);\n    for (const feature of features) {\n      const docsForFeature: ExtractedDoc[] = parsedDocs.map((doc) => ({ path: doc.relPath, extracted: doc.extracted[feature.name] }));\n      await feature.store?.(conn, docsForFeature, delta);\n    }\n    for (const feature of features) await feature.afterReconcile?.(conn, delta);\n  });\n\n  return { parsed: parsedDocs.length, warnings };\n}\n"],"names":["SenseError","activeFeatures","progress","listFiles","RESERVED_COLUMNS","reparseFiles","getColumns","quoteIdent","withTransaction","CORE_FRONTMATTER_COLUMNS","Set","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcile","conn","cfg","baseDir","files","currentSet","map","f","existingStmt","prepare","existingRows","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","report","docs","parsedDocs","newColumns","tick","finish","col","add","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","addedSet","reparsedExisting","p","feature","exec","rows","ctimeMs","parseError","data","runBatch","contentTouched","presetTouched","presetRows","presetName","presets","push","removedPaths","remove","docsForFeature","extracted","name","store","afterReconcile"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,cAAc,QAAQ,0BAA0B;AAEzD,SAASC,QAAQ,QAAQ,2BAA2B;AAEpD,SAASC,SAAS,EAAEC,gBAAgB,QAAQ,sBAAsB;AAClE,SAASC,YAAY,QAAQ,wBAAwB;AACrD,SAASC,UAAU,EAAEC,UAAU,QAAQ,eAAe;AACtD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,4FAA4F;AAC5F,8FAA8F;AAC9F,+FAA+F;AAC/F,iGAAiG;AACjG,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,iBAAiB;AACjB,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,6FAA6F;AAC7F,2FAA2F;AAC3F,8BAA8B;AAC9B,MAAMC,0BAA0B;AAEhC,OAAO,eAAeC,UAAUC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5E,MAAMC,QAAQrB,UAAUmB,KAAKC;IAC7B,MAAME,aAAa,IAAIf,IAAIc,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IAErD,MAAMc,eAAe,MAAMP,KAAKQ,OAAO,CAAC;IACxC,MAAMC,eAAgB,MAAMF,aAAaG,GAAG;IAC5C,MAAMC,WAAW,IAAIC,IAAIH,aAAaJ,GAAG,CAAC,CAACQ,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWN,aAAaO,MAAM,CAAC,CAACH,IAAM,CAACT,WAAWa,GAAG,CAACJ,EAAEC,IAAI,GAAGT,GAAG,CAAC,CAACQ,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYf,MAAMa,MAAM,CAAC,CAACV;QAC9B,MAAMa,MAAMR,SAASS,GAAG,CAACd,EAAEb,OAAO;QAClC,OAAO,CAAC0B,OAAOA,IAAIE,MAAM,KAAKf,EAAEgB,OAAO,IAAIH,IAAII,KAAK,KAAKjB,EAAEkB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWhD,eAAeqB;IAChC,MAAM4B,cAAc,MAAM5C,WAAWe;IAErC,MAAM8B,SAASjD,SAAS,mBAAmBqC,UAAUO,MAAM;IAC3D,MAAM,EAAEM,MAAMC,UAAU,EAAEL,QAAQ,EAAEM,UAAU,EAAE,GAAGjD,aAAakC,WAAWU,UAAU3B,KAAK4B,aAAaC,OAAOI,IAAI;IAClHJ,OAAOK,MAAM;IACb,KAAK,MAAMC,OAAOH,WAAYJ,YAAYQ,GAAG,CAACD;IAE9C,MAAME,aAAa;WAAIT;KAAY;IACnC,IAAIS,WAAWb,MAAM,GAAG3B,yBAAyB;QAC/C,MAAM,IAAInB,WAAW,gBAAgB,CAAC,uBAAuB,EAAE2D,WAAWb,MAAM,CAAC,8CAA8C,EAAE3B,wBAAwB,gIAAgI,CAAC;IAC5R;IACA,MAAMyC,kBAAkBD,WAAWtB,MAAM,CAAC,CAACwB,IAAMpD,yBAAyB6B,GAAG,CAACuB,MAAM,CAACzD,iBAAiBkC,GAAG,CAACuB;IAC1G,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgBlC,GAAG,CAACnB,YAAYwD,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgBlC,GAAG,CAAC,IAAM,KAAKqC,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLvB,MAAM,CAAC,CAACwB,IAAMA,MAAM,QACpBnC,GAAG,CAAC,CAACmC,IAAM,GAAGtD,WAAWsD,GAAG,YAAY,EAAEtD,WAAWsD,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQzB,UAAUF,MAAM,CAAC,CAACV,IAAM,CAACK,SAASM,GAAG,CAACX,EAAEb,OAAO,GAAGY,GAAG,CAAC,CAACC,IAAMA,EAAEb,OAAO;IACpF,MAAMmD,QAAwB;QAAEzC;QAAO0C,UAAUb,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO;QAAGkD;QAAO5B;IAAS;IACnG,MAAMgC,WAAW,IAAI1D,IAAIsD;IACzB,MAAMK,mBAAmBhB,WAAW3B,GAAG,CAAC,CAACyC,IAAMA,EAAErD,OAAO,EAAEuB,MAAM,CAAC,CAACiC,IAAM,CAACF,SAAS9B,GAAG,CAACgC;IAEtF,MAAM9D,gBAAgBa,MAAM;YA8CyCkD,iBAK7BA;QAlDtC,yFAAyF;QACzF,2FAA2F;QAC3F,KAAK,MAAMd,OAAOH,WAAY,MAAMjC,KAAKmD,IAAI,CAAC,CAAC,mCAAmC,EAAEjE,WAAWkD,KAAK,QAAQ,CAAC;QAE7G,IAAIJ,WAAWP,MAAM,GAAG,GAAG;YACzB,MAAM2B,OAAOpB,WAAW3B,GAAG,CAAC,CAACb,MAC3B+C,gBAAgBlC,GAAG,CAAC,CAAC+B;wBAMZ5C;oBALP,IAAI4C,QAAQ,QAAQ,OAAO5C,IAAIC,OAAO;oBACtC,IAAI2C,QAAQ,UAAU,OAAO5C,IAAI8B,OAAO;oBACxC,IAAIc,QAAQ,UAAU,OAAO5C,IAAI6D,OAAO;oBACxC,IAAIjB,QAAQ,SAAS,OAAO5C,IAAIgC,IAAI;oBACpC,IAAIY,QAAQ,gBAAgB,OAAO5C,IAAI8D,UAAU;oBACjD,QAAO9D,gBAAAA,IAAI+D,IAAI,CAACnB,IAAI,cAAb5C,2BAAAA,gBAAiB;gBAC1B;YAEF,MAAMQ,KAAKwD,QAAQ,CAACf,WAAWW;QACjC;QAEA,MAAMK,iBAAiB;eAAI1C;eAAaiC;SAAiB;QACzD,IAAIS,eAAehC,MAAM,GAAG,GAC1B,MAAMzB,KAAKwD,QAAQ,CACjB,wCACAC,eAAepD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEjC,IAAIjB,WAAWP,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAClE,oBAAoB0C,WAAW3B,GAAG,CAACd;QAElF,IAAIwB,SAASU,MAAM,GAAG,GACpB,MAAMzB,KAAKwD,QAAQ,CACjB,4CACAzC,SAASV,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAG3B,0FAA0F;QAC1F,kEAAkE;QAClE,MAAMS,gBAAgB;eAAI3C;eAAaiC;SAAiB;QACxD,IAAIU,cAAcjC,MAAM,GAAG,GACzB,MAAMzB,KAAKwD,QAAQ,CACjB,6CACAE,cAAcrD,GAAG,CAAC,CAAC4C,IAAM;gBAACA;aAAE;QAEhC,MAAMU,aAA0B,EAAE;QAClC,KAAK,MAAMnE,OAAOwC,WAAY,KAAK,MAAM4B,cAAcpE,IAAIqE,OAAO,CAAEF,WAAWG,IAAI,CAAC;YAACtE,IAAIC,OAAO;YAAEmE;SAAW;QAC7G,IAAID,WAAWlC,MAAM,GAAG,GAAG,MAAMzB,KAAKwD,QAAQ,CAAC,2DAA2DG;QAE1G,MAAMI,eAAe;eAAIhD;eAAaiC;SAAiB;QACvD,IAAIe,aAAatC,MAAM,GAAG,GAAG,KAAK,MAAMyB,WAAWtB,SAAU,QAAMsB,kBAAAA,QAAQc,MAAM,cAAdd,sCAAAA,qBAAAA,SAAiBlD,MAAM+D,cAAcnB;QACxG,KAAK,MAAMM,WAAWtB,SAAU;gBAExBsB;YADN,MAAMe,iBAAiCjC,WAAW3B,GAAG,CAAC,CAACb,MAAS,CAAA;oBAAEsB,MAAMtB,IAAIC,OAAO;oBAAEyE,WAAW1E,IAAI0E,SAAS,CAAChB,QAAQiB,IAAI,CAAC;gBAAC,CAAA;YAC5H,QAAMjB,iBAAAA,QAAQkB,KAAK,cAAblB,qCAAAA,oBAAAA,SAAgBlD,MAAMiE,gBAAgBrB;QAC9C;QACA,KAAK,MAAMM,WAAWtB,SAAU,QAAMsB,0BAAAA,QAAQmB,cAAc,cAAtBnB,8CAAAA,6BAAAA,SAAyBlD,MAAM4C;IACvE;IAEA,OAAO;QAAElB,QAAQM,WAAWP,MAAM;QAAEE;IAAS;AAC/C"}

package/dist/esm/store/duckdb/sql-functions.d.ts

@@ -0,0 +1,2 @@
+import type { DuckDBConnection } from '@duckdb/node-api';
+export declare function registerFunctions(conn: DuckDBConnection): Promise<void>;

package/dist/esm/store/duckdb/sql-functions.js

@@ -0,0 +1,56 @@
+import { segmentMatch } from '../../text/segment.js';
+import { basenameImpl, hasImpl } from '../sql-functions.js';
+import { duckdbApi } from './native.js';
+// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional
+// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a
+// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already
+// resolved the package, possibly via an on-demand install, and a second independent bare-specifier
+// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).
+export async function registerFunctions(conn) {
+    const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();
+    conn.registerScalarFunction(DuckDBScalarFunction.create({
+        name: 'has',
+        parameterTypes: [
+            DuckDBVarCharType.instance,
+            DuckDBVarCharType.instance
+        ],
+        returnType: DuckDBIntegerType.instance,
+        mainFunction (_info, inputChunk, outputVector) {
+            const fieldVec = inputChunk.getColumnVector(0);
+            const valueVec = inputChunk.getColumnVector(1);
+            for(let i = 0; i < inputChunk.rowCount; i++)outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));
+            outputVector.flush();
+        }
+    }));
+    conn.registerScalarFunction(DuckDBScalarFunction.create({
+        name: 'basename',
+        parameterTypes: [
+            DuckDBVarCharType.instance
+        ],
+        varArgsType: DuckDBVarCharType.instance,
+        returnType: DuckDBVarCharType.instance,
+        mainFunction (_info, inputChunk, outputVector) {
+            const pathVec = inputChunk.getColumnVector(0);
+            const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;
+            for(let i = 0; i < inputChunk.rowCount; i++)outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));
+            outputVector.flush();
+        }
+    }));
+    // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars
+    // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)
+    // and never calls this, but hand-written raw SQL naming it still gets the real implementation
+    // rather than a silent passthrough (principle 6).
+    conn.registerScalarFunction(DuckDBScalarFunction.create({
+        name: 'segment',
+        parameterTypes: [
+            DuckDBVarCharType.instance
+        ],
+        returnType: DuckDBVarCharType.instance,
+        mainFunction (_info, inputChunk, outputVector) {
+            var _textVec_getItem;
+            const textVec = inputChunk.getColumnVector(0);
+            for(let i = 0; i < inputChunk.rowCount; i++)outputVector.setItem(i, segmentMatch(String((_textVec_getItem = textVec.getItem(i)) !== null && _textVec_getItem !== void 0 ? _textVec_getItem : '')));
+            outputVector.flush();
+        }
+    }));
+}

package/dist/esm/store/duckdb/sql-functions.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional\n// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a\n// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already\n// resolved the package, possibly via an on-demand install, and a second independent bare-specifier\n// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars\n  // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)\n  // and never calls this, but hand-written raw SQL naming it still gets the real implementation\n  // rather than a silent passthrough (principle 6).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,iGAAiG;AACjG,gGAAgG;AAChG,6FAA6F;AAC7F,mGAAmG;AACnG,iGAAiG;AACjG,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,0FAA0F;IAC1F,8FAA8F;IAC9F,8FAA8F;IAC9F,kDAAkD;IAClDrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}

package/dist/esm/store/duckdb/store.d.ts

@@ -0,0 +1,5 @@
+import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';
+import type { Config } from '../../config/index.js';
+import type { Capability, Connection, Store } from '../types.js';
+export declare const CAPABILITIES: ReadonlySet<Capability>;
+export declare function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store;

package/dist/esm/store/duckdb/store.js

@@ -0,0 +1,104 @@
+import { STORE_DIMS } from '../../embed/types.js';
+import { getColumns } from '../shared.js';
+// pending()/hasVector() are IS NULL/IS NOT NULL checks the portable Connection already
+// answers identically regardless of the vector column's type, so this store reuses sqlite's
+// (engine-neutral despite the module name) instead of duplicating them.
+import { hasVectorRow, pendingRows } from '../sqlite/vectors.js';
+import { withTransaction } from '../transaction.js';
+import { storeRowToJs } from './connection.js';
+import { createLexicalIndex } from './lexical.js';
+import { reconcile } from './reconcile.js';
+import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
+// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +
+// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see
+// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every
+// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a
+// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):
+// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it
+// rather than answer it differently.
+export const CAPABILITIES = new Set([
+    'lexical',
+    'phrases',
+    'vectors'
+]);
+// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth
+// (see transaction.ts) is tracked against the same object everywhere. The instance is the
+// native database handle that owns the WAL: close() must close it, not just disconnect the
+// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.
+export function createStore(instance, duckdb, conn, cfg, baseDir) {
+    const lex = createLexicalIndex(conn);
+    return {
+        name: 'duckdb',
+        capabilities: CAPABILITIES,
+        async exec (sql) {
+            await conn.exec(sql);
+        },
+        async prepare (sql) {
+            return conn.prepare(sql);
+        },
+        async runBatch (sql, paramRows) {
+            await conn.runBatch(sql, paramRows);
+        },
+        async transaction (fn) {
+            return withTransaction(conn, fn);
+        },
+        async reconcile () {
+            const result = await reconcile(conn, cfg, baseDir);
+            // content may have changed; the fts index is rebuilt lazily, on the next lexical query
+            // that needs it, not here (see lexical.ts's FtsIndexState).
+            lex.markStale();
+            return result;
+        },
+        docs: {
+            async columns () {
+                return [
+                    ...await getColumns(conn)
+                ];
+            }
+        },
+        lexical: {
+            query: lex.query
+        },
+        // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the
+        // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is
+        // still correct against a wider column.
+        vectors: {
+            pending: ()=>pendingRows(conn),
+            writeVectors: (rows)=>writeVectorBatch(duckdb, conn, STORE_DIMS, rows),
+            candidates: (qv, _storeDims, fetch, allowed)=>scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),
+            similar: (path, opts)=>scanSimilar(duckdb, conn, STORE_DIMS, path, opts),
+            hasVector: (path)=>hasVectorRow(conn, path)
+        },
+        raw: {
+            async prepare (sql) {
+                const prepared = await duckdb.prepare(sql);
+                return {
+                    columns: ()=>{
+                        const out = [];
+                        for(let i = 0; i < prepared.columnCount; i++)out.push({
+                            name: prepared.columnName(i)
+                        });
+                        return out;
+                    },
+                    // Real incremental streaming: DuckDB reads its own result in row-group chunks, so
+                    // each readUntil only fetches what isn't already in hand.
+                    iterate: async function*(...params) {
+                        if (params.length > 0) prepared.bind(params);
+                        const reader = await prepared.streamAndRead();
+                        let yielded = 0;
+                        do {
+                            await reader.readUntil(reader.currentRowCount + 2048);
+                            const rows = reader.getRowObjectsJS();
+                            for(; yielded < rows.length; yielded++)yield storeRowToJs(rows[yielded]);
+                        }while (!reader.done)
+                    }
+                };
+            }
+        },
+        async close () {
+            // Order matters: the connection must be gone before the instance closes the WAL.
+            duckdb.disconnectSync();
+            instance.closeSync();
+        }
+    };
+}

package/dist/esm/store/duckdb/store.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport type { Config } from '../../config/index.ts';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\n// pending()/hasVector() are IS NULL/IS NOT NULL checks the portable Connection already\n// answers identically regardless of the vector column's type, so this store reuses sqlite's\n// (engine-neutral despite the module name) instead of duplicating them.\nimport { hasVectorRow, pendingRows } from '../sqlite/vectors.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { reconcile } from './reconcile.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (D1: fts BM25 +\n// contains(), see lexical.ts), and vectors (D2: native FLOAT[N] + array_cosine_similarity, see\n// vectors.ts) are implemented. 'snippets' is declined: no snippet()-equivalent exists, so every\n// lexical hit relies on the caller's JS excerpt fallback (commands/search.ts) rather than a\n// bounded engine-native one. 'phrases' means quoted-phrase matching only (types.ts):\n// prefix/boolean/NEAR/column-filter FTS5 syntax is not claimed here -- lexical.ts rejects it\n// rather than answer it differently.\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['lexical', 'phrases', 'vectors']);\n\n// Shares one Connection instance (conn) with open()'s own reconcile call so transaction depth\n// (see transaction.ts) is tracked against the same object everywhere. The instance is the\n// native database handle that owns the WAL: close() must close it, not just disconnect the\n// connection, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection, cfg: Config, baseDir: string): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    async reconcile() {\n      const result = await reconcile(conn, cfg, baseDir);\n      // content may have changed; the fts index is rebuilt lazily, on the next lexical query\n      // that needs it, not here (see lexical.ts's FtsIndexState).\n      lex.markStale();\n      return result;\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the\n    // interface's per-call storeDims -- see vectors.ts's padded() for why a shorter vector is\n    // still correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["STORE_DIMS","getColumns","hasVectorRow","pendingRows","withTransaction","storeRowToJs","createLexicalIndex","reconcile","scanCandidates","scanSimilar","writeVectorBatch","CAPABILITIES","Set","createStore","instance","duckdb","conn","cfg","baseDir","lex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","result","markStale","docs","columns","lexical","query","vectors","pending","writeVectors","rows","candidates","qv","_storeDims","fetch","allowed","similar","path","opts","hasVector","raw","prepared","out","i","columnCount","push","columnName","iterate","params","length","bind","reader","streamAndRead","yielded","readUntil","currentRowCount","getRowObjectsJS","done","close","disconnectSync","closeSync"],"mappings":"AAEA,SAASA,UAAU,QAAQ,uBAAuB;AAClD,SAASC,UAAU,QAAQ,eAAe;AAC1C,uFAAuF;AACvF,4FAA4F;AAC5F,wEAAwE;AACxE,SAASC,YAAY,EAAEC,WAAW,QAAQ,uBAAuB;AACjE,SAASC,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,kBAAkB;AAC/C,SAASC,kBAAkB,QAAQ,eAAe;AAClD,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,cAAc,EAAEC,WAAW,EAAEC,gBAAgB,QAAQ,eAAe;AAE7E,2FAA2F;AAC3F,+FAA+F;AAC/F,gGAAgG;AAChG,4FAA4F;AAC5F,qFAAqF;AACrF,6FAA6F;AAC7F,qCAAqC;AACrC,OAAO,MAAMC,eAAwC,IAAIC,IAAI;IAAC;IAAW;IAAW;CAAU,EAAE;AAEhG,8FAA8F;AAC9F,0FAA0F;AAC1F,2FAA2F;AAC3F,oFAAoF;AACpF,OAAO,SAASC,YAAYC,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC5H,MAAMC,MAAMb,mBAAmBU;IAC/B,OAAO;QACLI,MAAM;QACNC,cAAcV;QACd,MAAMW,MAAKC,GAAW;YACpB,MAAMP,KAAKM,IAAI,CAACC;QAClB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAOP,KAAKQ,OAAO,CAACD;QACtB;QACA,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,MAAMV,KAAKS,QAAQ,CAACF,KAAKG;QAC3B;QACA,MAAMC,aAAeC,EAAoB;YACvC,OAAOxB,gBAAgBY,MAAMY;QAC/B;QACA,MAAMrB;YACJ,MAAMsB,SAAS,MAAMtB,UAAUS,MAAMC,KAAKC;YAC1C,uFAAuF;YACvF,4DAA4D;YAC5DC,IAAIW,SAAS;YACb,OAAOD;QACT;QACAE,MAAM;YACJ,MAAMC;gBACJ,OAAO;uBAAK,MAAM/B,WAAWe;iBAAO;YACtC;QACF;QACAiB,SAAS;YACPC,OAAOf,IAAIe,KAAK;QAClB;QACA,sFAAsF;QACtF,0FAA0F;QAC1F,wCAAwC;QACxCC,SAAS;YACPC,SAAS,IAAMjC,YAAYa;YAC3BqB,cAAc,CAACC,OAAS5B,iBAAiBK,QAAQC,MAAMhB,YAAYsC;YACnEC,YAAY,CAACC,IAAIC,YAAYC,OAAOC,UAAYnC,eAAeO,QAAQyB,IAAIxC,YAAY0C,OAAOC;YAC9FC,SAAS,CAACC,MAAMC,OAASrC,YAAYM,QAAQC,MAAMhB,YAAY6C,MAAMC;YACrEC,WAAW,CAACF,OAAS3C,aAAac,MAAM6B;QAC1C;QACAG,KAAK;YACH,MAAMxB,SAAQD,GAAW;gBACvB,MAAM0B,WAAW,MAAMlC,OAAOS,OAAO,CAACD;gBACtC,OAAO;oBACLS,SAAS;wBACP,MAAMkB,MAA+B,EAAE;wBACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;4BAAEjC,MAAM6B,SAASK,UAAU,CAACH;wBAAG;wBACvF,OAAOD;oBACT;oBACA,kFAAkF;oBAClF,0DAA0D;oBAC1DK,SAAS,gBAAiB,GAAGC,MAAiB;wBAC5C,IAAIA,OAAOC,MAAM,GAAG,GAAGR,SAASS,IAAI,CAACF;wBACrC,MAAMG,SAAS,MAAMV,SAASW,aAAa;wBAC3C,IAAIC,UAAU;wBACd,GAAG;4BACD,MAAMF,OAAOG,SAAS,CAACH,OAAOI,eAAe,GAAG;4BAChD,MAAMzB,OAAOqB,OAAOK,eAAe;4BACnC,MAAOH,UAAUvB,KAAKmB,MAAM,EAAEI,UAAW,MAAMxD,aAAaiC,IAAI,CAACuB,QAAQ;wBAC3E,QAAS,CAACF,OAAOM,IAAI,CAAE;oBACzB;gBACF;YACF;QACF;QACA,MAAMC;YACJ,iFAAiF;YACjFnD,OAAOoD,cAAc;YACrBrD,SAASsD,SAAS;QACpB;IACF;AACF"}

package/dist/esm/store/duckdb/vectors.d.ts

@@ -0,0 +1,9 @@
+import type { DuckDBConnection } from '@duckdb/node-api';
+import type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.js';
+export declare function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void>;
+export declare function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export declare function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: {
+    exclude: Set<string>;
+    allowed?: Set<string>;
+    k: number;
+}): Promise<VectorSimilar[]>;

package/dist/esm/store/duckdb/vectors.js

@@ -0,0 +1,171 @@
+import { withTransaction } from '../transaction.js';
+import { rewriteUpdate } from './batch.js';
+import { duckdbApi } from './native.js';
+// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by
+// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with
+// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --
+// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own
+// representation and stays as it is. Every function here takes `dims` as a parameter rather
+// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real
+// constant at the call site).
+//
+// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only
+// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's
+// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever
+// open.ts already resolved, including via an on-demand install (see native.ts).
+// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only
+// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).
+const untyped = undefined;
+// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's
+// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's
+// based on -- 12.7s at 201 chunks/note without sampling).
+const TARGET_CHUNK_CAP = 16;
+// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly
+// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the
+// same bounded number rather than one having more decimal noise than the other.
+function asCosine(score) {
+    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
+}
+// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced
+// under the column's width); zero-padding leaves cosine scores unchanged since the added
+// dimensions contribute nothing to either vector's dot product or norm.
+function padded(values, dims) {
+    const out = new Array(dims).fill(0);
+    for(let d = 0; d < Math.min(values.length, dims); d++)out[d] = values[d];
+    return out;
+}
+function inClause(column, count) {
+    return `${column} IN (${Array.from({
+        length: count
+    }, ()=>'?').join(', ')})`;
+}
+// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces
+// for every store, see toStore); dequantizing into a plain float array is this store's own
+// representation choice, so both stores are handed the same vectors and diverge only here.
+function dequantize(row, dims) {
+    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
+    return padded(Array.from(q, (v)=>v * row.scale), dims);
+}
+// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's
+// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native
+// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --
+// automatic type inference on the raw values misreads an all-integer first component (a real
+// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row
+// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).
+export async function writeVectorBatch(duckdb, conn, dims, rows) {
+    if (rows.length === 0) return;
+    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
+    const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE "path" = ? AND chunk = ?', rows.length);
+    if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');
+    await withTransaction(conn, async ()=>{
+        const stmt = await duckdb.prepare(rewritten.sql);
+        try {
+            const values = [];
+            const types = [];
+            for (const row of rows){
+                values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);
+                types.push(ARRAY(DOUBLE, dims), untyped, untyped);
+            }
+            stmt.bind(values, types);
+            await stmt.run();
+        } finally{
+            stmt.destroySync();
+        }
+    });
+}
+// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks
+// the winning chunk's line range from the same row max(score) came from, per path. An empty
+// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.
+export async function scanCandidates(duckdb, qv, dims, fetch, allowed) {
+    if (allowed && allowed.size === 0) return [];
+    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
+    const allowedList = allowed ? [
+        ...allowed
+    ] : [];
+    const sql = `SELECT "path", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score
+    FROM (
+      SELECT "path", start_line, end_line, array_cosine_similarity(vector, ?) AS score
+      FROM embeddings
+      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('"path"', allowedList.length)}` : ''}
+    ) sub
+    GROUP BY "path"
+    ORDER BY score DESC
+    LIMIT ?`;
+    const stmt = await duckdb.prepare(sql);
+    try {
+        const values = [
+            arrayValue(padded(qv, dims)),
+            ...allowedList,
+            fetch
+        ];
+        const types = [
+            ARRAY(DOUBLE, dims),
+            ...allowedList.map(()=>untyped),
+            untyped
+        ];
+        stmt.bind(values, types);
+        const reader = await stmt.runAndReadAll();
+        const rows = reader.getRowObjectsJS();
+        return rows.map((r)=>({
+                path: r.path,
+                lines: `L${r.start_line}-${r.end_line}`,
+                similarity: asCosine(r.score)
+            }));
+    } finally{
+        stmt.destroySync();
+    }
+}
+// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed
+// into one native scan: the sampled target vectors become a small VALUES list, cross-joined
+// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost
+// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.
+export async function scanSimilar(duckdb, conn, dims, path, opts) {
+    if (opts.allowed && opts.allowed.size === 0) return [];
+    const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk');
+    const targetRows = await targetStmt.all(path);
+    if (targetRows.length === 0) return [];
+    const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
+    const targets = targetRows.filter((_, i)=>i % step === 0).map((row)=>row.vector);
+    const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();
+    const excludeList = [
+        ...opts.exclude
+    ];
+    const allowedList = opts.allowed ? [
+        ...opts.allowed
+    ] : [];
+    const sql = `WITH targets(tv) AS (VALUES ${targets.map(()=>'(?)').join(', ')})
+    SELECT e."path" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score
+    FROM embeddings e, targets t
+    WHERE e.vector IS NOT NULL AND e."path" != ?
+      ${excludeList.length > 0 ? `AND NOT ${inClause('e."path"', excludeList.length)}` : ''}
+      ${opts.allowed ? `AND ${inClause('e."path"', allowedList.length)}` : ''}
+    GROUP BY e."path"
+    ORDER BY score DESC
+    LIMIT ?`;
+    const stmt = await duckdb.prepare(sql);
+    try {
+        const values = [
+            ...targets.map((t)=>arrayValue(padded(t, dims))),
+            path,
+            ...excludeList,
+            ...allowedList,
+            opts.k
+        ];
+        const types = [
+            ...targets.map(()=>ARRAY(DOUBLE, dims)),
+            untyped,
+            ...excludeList.map(()=>untyped),
+            ...allowedList.map(()=>untyped),
+            untyped
+        ];
+        stmt.bind(values, types);
+        const reader = await stmt.runAndReadAll();
+        const rows = reader.getRowObjectsJS();
+        return rows.map((r)=>({
+                path: r.path,
+                similarity: asCosine(r.score)
+            }));
+    } finally{
+        stmt.destroySync();
+    }
+}

package/dist/esm/store/duckdb/vectors.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { rewriteUpdate } from './batch.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by\n// duckdb/open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and scans them with\n// array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS --\n// sqlite's int8+scale BLOB scan (src/store/sqlite/vectors.ts) is the other store's own\n// representation and stays as it is. Every function here takes `dims` as a parameter rather\n// than importing STORE_DIMS directly, so it is testable at any width (store.ts wires the real\n// constant at the call site).\n//\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only\n// tree would resolve this optional dependency just by importing store/index.ts -- native.ts's\n// shared duckdbApi() (not a bare import of its own) is what keeps that lazy and reuses whatever\n// open.ts already resolved, including via an on-demand install (see native.ts).\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\n\n// Seed chunks that participate in a `related` scan; must match sqlite/vectors.ts's\n// TARGET_CHUNK_CAP for identical sampling behavior (see that file for the measurement it's\n// based on -- 12.7s at 201 chunks/note without sampling).\nconst TARGET_CHUNK_CAP = 16;\n\n// array_cosine_similarity is already a true cosine (no int8 accumulation to push slightly\n// outside [-1, 1]), but rounded the same way sqlite's scores are so both stores print the\n// same bounded number rather than one having more decimal noise than the other.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced\n// under the column's width); zero-padding leaves cosine scores unchanged since the added\n// dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces\n// for every store, see toStore); dequantizing into a plain float array is this store's own\n// representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// One runBatch-equivalent call per provider batch: a single multi-row UPDATE (batch.ts's\n// rewriteUpdate, the same rewrite Connection.runBatch uses), bound directly against the native\n// connection because the vector column needs an explicit ARRAY(DOUBLE, dims) bind type --\n// automatic type inference on the raw values misreads an all-integer first component (a real\n// vector's exact 0.0 is common) as an INTEGER array and silently truncates the rest of the row\n// (verified against a live connection: an untyped bind of [0, 0.35, 0.1, -0.2] stores [0,0,0,0]).\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const rewritten = rewriteUpdate('UPDATE embeddings SET vector = ? WHERE \"path\" = ? AND chunk = ?', rows.length);\n  if (!rewritten) throw new Error('duckdb: writeVectorBatch SQL shape not recognized by rewriteUpdate');\n\n  await withTransaction(conn, async () => {\n    const stmt = await duckdb.prepare(rewritten.sql);\n    try {\n      const values: DuckDBValue[] = [];\n      const types: DuckDBType[] = [];\n      for (const row of rows) {\n        values.push(arrayValue(dequantize(row, dims)), row.path, row.chunk);\n        types.push(ARRAY(DOUBLE, dims), untyped, untyped);\n      }\n      stmt.bind(values, types);\n      await stmt.run();\n    } finally {\n      stmt.destroySync();\n    }\n  });\n}\n\n// Best chunk per file by cosine, its line range riding along, entirely in SQL: arg_max picks\n// the winning chunk's line range from the same row max(score) came from, per path. An empty\n// `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const sql = `SELECT \"path\", arg_max(start_line, score) AS start_line, arg_max(end_line, score) AS end_line, max(score) AS score\n    FROM (\n      SELECT \"path\", start_line, end_line, array_cosine_similarity(vector, ?) AS score\n      FROM embeddings\n      WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n    ) sub\n    GROUP BY \"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [arrayValue(padded(qv, dims)), ...allowedList, fetch];\n    const types: DuckDBType[] = [ARRAY(DOUBLE, dims), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed\n// into one native scan: the sampled target vectors become a small VALUES list, cross-joined\n// against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost\n// the plan measures (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const targets = targetRows.filter((_, i) => i % step === 0).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')})\n    SELECT e.\"path\" AS path, max(array_cosine_similarity(e.vector, t.tv)) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n    ORDER BY score DESC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["withTransaction","rewriteUpdate","duckdbApi","untyped","undefined","TARGET_CHUNK_CAP","asCosine","score","Math","round","min","max","padded","values","dims","out","Array","fill","d","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","writeVectorBatch","duckdb","conn","rows","arrayValue","ARRAY","DOUBLE","rewritten","Error","stmt","prepare","sql","types","push","path","chunk","bind","run","destroySync","scanCandidates","qv","fetch","allowed","size","allowedList","map","reader","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","scanSimilar","opts","targetStmt","targetRows","all","step","ceil","targets","filter","_","i","excludeList","exclude","t","k"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,aAAa,QAAQ,aAAa;AAC3C,SAASC,SAAS,QAAQ,cAAc;AAExC,mFAAmF;AACnF,uFAAuF;AACvF,0FAA0F;AAC1F,uFAAuF;AACvF,4FAA4F;AAC5F,8FAA8F;AAC9F,8BAA8B;AAC9B,EAAE;AACF,4FAA4F;AAC5F,8FAA8F;AAC9F,gGAAgG;AAChG,gFAAgF;AAEhF,6FAA6F;AAC7F,6FAA6F;AAC7F,MAAMC,UAAUC;AAEhB,mFAAmF;AACnF,2FAA2F;AAC3F,0DAA0D;AAC1D,MAAMC,mBAAmB;AAEzB,0FAA0F;AAC1F,0FAA0F;AAC1F,gFAAgF;AAChF,SAASC,SAASC,KAAa;IAC7B,OAAOC,KAAKC,KAAK,CAACD,KAAKE,GAAG,CAAC,GAAGF,KAAKG,GAAG,CAAC,CAAC,GAAGJ,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,wEAAwE;AACxE,SAASK,OAAOC,MAAyB,EAAEC,IAAY;IACrD,MAAMC,MAAM,IAAIC,MAAcF,MAAMG,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIV,KAAKE,GAAG,CAACG,OAAOM,MAAM,EAAEL,OAAOI,IAAKH,GAAG,CAACG,EAAE,GAAGL,MAAM,CAACK,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASK,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,GAAGD,OAAO,KAAK,EAAEL,MAAMO,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG,IAAM,KAAKE,IAAI,CAAC,MAAM,CAAC,CAAC;AAChF;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,2FAA2F;AAC3F,SAASC,WAAWC,GAAmB,EAAEZ,IAAY;IACnD,MAAMa,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOpB,OACLI,MAAMO,IAAI,CAACI,GAAG,CAACM,IAAMA,IAAIP,IAAIQ,KAAK,GAClCpB;AAEJ;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,0FAA0F;AAC1F,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,eAAeqB,iBAAiBC,MAAwB,EAAEC,IAAgB,EAAEvB,IAAY,EAAEwB,IAAsB;IACrH,IAAIA,KAAKnB,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEoB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAMwC,YAAYzC,cAAc,mEAAmEqC,KAAKnB,MAAM;IAC9G,IAAI,CAACuB,WAAW,MAAM,IAAIC,MAAM;IAEhC,MAAM3C,gBAAgBqC,MAAM;QAC1B,MAAMO,OAAO,MAAMR,OAAOS,OAAO,CAACH,UAAUI,GAAG;QAC/C,IAAI;YACF,MAAMjC,SAAwB,EAAE;YAChC,MAAMkC,QAAsB,EAAE;YAC9B,KAAK,MAAMrB,OAAOY,KAAM;gBACtBzB,OAAOmC,IAAI,CAACT,WAAWd,WAAWC,KAAKZ,QAAQY,IAAIuB,IAAI,EAAEvB,IAAIwB,KAAK;gBAClEH,MAAMC,IAAI,CAACR,MAAMC,QAAQ3B,OAAOX,SAASA;YAC3C;YACAyC,KAAKO,IAAI,CAACtC,QAAQkC;YAClB,MAAMH,KAAKQ,GAAG;QAChB,SAAU;YACRR,KAAKS,WAAW;QAClB;IACF;AACF;AAEA,6FAA6F;AAC7F,4FAA4F;AAC5F,2FAA2F;AAC3F,OAAO,eAAeC,eAAelB,MAAwB,EAAEmB,EAAgB,EAAEzC,IAAY,EAAE0C,KAAa,EAAEC,OAAqB;IACjI,IAAIA,WAAWA,QAAQC,IAAI,KAAK,GAAG,OAAO,EAAE;IAC5C,MAAM,EAAEnB,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAMyD,cAAcF,UAAU;WAAIA;KAAQ,GAAG,EAAE;IAC/C,MAAMX,MAAM,CAAC;;;;+BAIgB,EAAEW,UAAU,CAAC,IAAI,EAAErC,SAAS,UAAUuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;;WAIrF,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMjC,SAAwB;YAAC0B,WAAW3B,OAAO2C,IAAIzC;eAAW6C;YAAaH;SAAM;QACnF,MAAMT,QAAsB;YAACP,MAAMC,QAAQ3B;eAAU6C,YAAYC,GAAG,CAAC,IAAMzD;YAAUA;SAAQ;QAC7FyC,KAAKO,IAAI,CAACtC,QAAQkC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEgB,OAAO,CAAC,CAAC,EAAED,EAAEE,UAAU,CAAC,CAAC,EAAEF,EAAEG,QAAQ,EAAE;gBAAEC,YAAY9D,SAAS0D,EAAEzD,KAAK;YAAE,CAAA;IACjH,SAAU;QACRqC,KAAKS,WAAW;IAClB;AACF;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,8FAA8F;AAC9F,wFAAwF;AACxF,OAAO,eAAegB,YAAYjC,MAAwB,EAAEC,IAAgB,EAAEvB,IAAY,EAAEmC,IAAY,EAAEqB,IAAgE;IACxK,IAAIA,KAAKb,OAAO,IAAIa,KAAKb,OAAO,CAACC,IAAI,KAAK,GAAG,OAAO,EAAE;IACtD,MAAMa,aAAa,MAAMlC,KAAKQ,OAAO,CAAC;IACtC,MAAM2B,aAAc,MAAMD,WAAWE,GAAG,CAACxB;IACzC,IAAIuB,WAAWrD,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMuD,OAAOlE,KAAKG,GAAG,CAAC,GAAGH,KAAKmE,IAAI,CAACH,WAAWrD,MAAM,GAAGd;IACvD,MAAMuE,UAAUJ,WAAWK,MAAM,CAAC,CAACC,GAAGC,IAAMA,IAAIL,SAAS,GAAGd,GAAG,CAAC,CAAClC,MAAQA,IAAIG,MAAM;IAEnF,MAAM,EAAEU,UAAU,EAAEC,KAAK,EAAEC,MAAM,EAAE,GAAG,MAAMvC;IAC5C,MAAM8E,cAAc;WAAIV,KAAKW,OAAO;KAAC;IACrC,MAAMtB,cAAcW,KAAKb,OAAO,GAAG;WAAIa,KAAKb,OAAO;KAAC,GAAG,EAAE;IACzD,MAAMX,MAAM,CAAC,4BAA4B,EAAE8B,QAAQhB,GAAG,CAAC,IAAM,OAAOpC,IAAI,CAAC,MAAM;;;;MAI3E,EAAEwD,YAAY7D,MAAM,GAAG,IAAI,CAAC,QAAQ,EAAEC,SAAS,YAAY4D,YAAY7D,MAAM,GAAG,GAAG,GAAG;MACtF,EAAEmD,KAAKb,OAAO,GAAG,CAAC,IAAI,EAAErC,SAAS,YAAYuC,YAAYxC,MAAM,GAAG,GAAG,GAAG;;;WAGnE,CAAC;IACV,MAAMyB,OAAO,MAAMR,OAAOS,OAAO,CAACC;IAClC,IAAI;QACF,MAAMjC,SAAwB;eAAI+D,QAAQhB,GAAG,CAAC,CAACsB,IAAM3C,WAAW3B,OAAOsE,GAAGpE;YAASmC;eAAS+B;eAAgBrB;YAAaW,KAAKa,CAAC;SAAC;QAChI,MAAMpC,QAAsB;eAAI6B,QAAQhB,GAAG,CAAC,IAAMpB,MAAMC,QAAQ3B;YAAQX;eAAY6E,YAAYpB,GAAG,CAAC,IAAMzD;eAAawD,YAAYC,GAAG,CAAC,IAAMzD;YAAUA;SAAQ;QAC/JyC,KAAKO,IAAI,CAACtC,QAAQkC;QAClB,MAAMc,SAAS,MAAMjB,KAAKkB,aAAa;QACvC,MAAMxB,OAAOuB,OAAOE,eAAe;QACnC,OAAOzB,KAAKsB,GAAG,CAAC,CAACI,IAAO,CAAA;gBAAEf,MAAMe,EAAEf,IAAI;gBAAEmB,YAAY9D,SAAS0D,EAAEzD,KAAK;YAAE,CAAA;IACxE,SAAU;QACRqC,KAAKS,WAAW;IAClB;AACF"}

package/dist/esm/store/index.d.ts

@@ -0,0 +1,7 @@
+import type { ResolvedConfig } from '../config/index.js';
+import type { OpenResult } from './sqlite/open.js';
+export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
+export { getMeta, setMeta } from './meta.js';
+export type { OpenResult } from './sqlite/open.js';
+export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';
+export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';

package/dist/esm/store/index.js

@@ -0,0 +1,35 @@
+// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything
+// beyond frontmatter + content lives in src/features/.
+import { anyPresetEmbeds, storeName } from '../config/index.js';
+import { SenseError } from '../errors.js';
+import { openDuckdb } from './duckdb/open.js';
+import { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.js';
+import { openSqlite } from './sqlite/open.js';
+import { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.js';
+// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile
+// work for a config that cannot be satisfied), then the store itself opens.
+const REGISTRY = {
+    sqlite: {
+        capabilities: SQLITE_CAPABILITIES,
+        open: openSqlite
+    },
+    duckdb: {
+        capabilities: DUCKDB_CAPABILITIES,
+        open: openDuckdb
+    }
+};
+export async function openStore(cfg) {
+    const name = storeName(cfg);
+    const entry = REGISTRY[name];
+    if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store "${name}"; available: ${Object.keys(REGISTRY).join(', ')}`);
+    // The one unambiguous, config-level capability need: an embed block some preset actually
+    // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level
+    // (every preset defaults to wanting "words"), so a store lacking 'lexical' fails loudly at
+    // first use instead (see duckdb/store.ts's lexical.query()).
+    if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {
+        throw new SenseError('STORE_CAPABILITY_MISSING', `store "${name}" does not implement "vectors" in this build, but this config's "embed" block is in use by at least one preset; remove or narrow it, or set "store" to a store that supports vectors (sqlite)`);
+    }
+    return entry.open(cfg);
+}
+export { getMeta, setMeta } from './meta.js';
+export { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.js';

package/dist/esm/store/index.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/index.ts"],"sourcesContent":["// Rows -> a backing store: registry + openStore(cfg). Parsing lives in scan.ts; everything\n// beyond frontmatter + content lives in src/features/.\n\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { anyPresetEmbeds, storeName } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { openDuckdb } from './duckdb/open.ts';\nimport { CAPABILITIES as DUCKDB_CAPABILITIES } from './duckdb/store.ts';\nimport type { OpenResult } from './sqlite/open.ts';\nimport { openSqlite } from './sqlite/open.ts';\nimport { CAPABILITIES as SQLITE_CAPABILITIES } from './sqlite/store.ts';\nimport type { Capability } from './types.ts';\n\ntype StoreName = 'sqlite' | 'duckdb';\n\ninterface StoreEntry {\n  capabilities: ReadonlySet<Capability>;\n  open: (cfg: ResolvedConfig) => Promise<OpenResult>;\n}\n\n// Capabilities are checked against the registry entry before opening (no wasted connect/reconcile\n// work for a config that cannot be satisfied), then the store itself opens.\nconst REGISTRY: Record<StoreName, StoreEntry> = {\n  sqlite: { capabilities: SQLITE_CAPABILITIES, open: openSqlite },\n  duckdb: { capabilities: DUCKDB_CAPABILITIES, open: openDuckdb },\n};\n\nexport async function openStore(cfg: ResolvedConfig): Promise<OpenResult> {\n  const name = storeName(cfg);\n  const entry = REGISTRY[name];\n  if (!entry) throw new SenseError('STORE_UNKNOWN', `unknown backing store \"${name}\"; available: ${Object.keys(REGISTRY).join(', ')}`);\n\n  // The one unambiguous, config-level capability need: an embed block some preset actually\n  // uses for vectors. Word/lexical search has no equivalent unambiguous signal at this level\n  // (every preset defaults to wanting \"words\"), so a store lacking 'lexical' fails loudly at\n  // first use instead (see duckdb/store.ts's lexical.query()).\n  if (anyPresetEmbeds(cfg) && !entry.capabilities.has('vectors')) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"${name}\" does not implement \"vectors\" in this build, but this config's \"embed\" block is in use by at least one preset; remove or narrow it, or set \"store\" to a store that supports vectors (sqlite)`);\n  }\n\n  return entry.open(cfg);\n}\n\nexport { getMeta, setMeta } from './meta.ts';\nexport type { OpenResult } from './sqlite/open.ts';\nexport { clearCache, DB_FILENAME, docCount, SCHEMA_VERSION } from './sqlite/open.ts';\nexport type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.ts';\n"],"names":["anyPresetEmbeds","storeName","SenseError","openDuckdb","CAPABILITIES","DUCKDB_CAPABILITIES","openSqlite","SQLITE_CAPABILITIES","REGISTRY","sqlite","capabilities","open","duckdb","openStore","cfg","name","entry","Object","keys","join","has","getMeta","setMeta","clearCache","DB_FILENAME","docCount","SCHEMA_VERSION"],"mappings":"AAAA,2FAA2F;AAC3F,uDAAuD;AAGvD,SAASA,eAAe,EAAEC,SAAS,QAAQ,qBAAqB;AAChE,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASC,gBAAgBC,mBAAmB,QAAQ,oBAAoB;AAExE,SAASC,UAAU,QAAQ,mBAAmB;AAC9C,SAASF,gBAAgBG,mBAAmB,QAAQ,oBAAoB;AAUxE,kGAAkG;AAClG,4EAA4E;AAC5E,MAAMC,WAA0C;IAC9CC,QAAQ;QAAEC,cAAcH;QAAqBI,MAAML;IAAW;IAC9DM,QAAQ;QAAEF,cAAcL;QAAqBM,MAAMR;IAAW;AAChE;AAEA,OAAO,eAAeU,UAAUC,GAAmB;IACjD,MAAMC,OAAOd,UAAUa;IACvB,MAAME,QAAQR,QAAQ,CAACO,KAAK;IAC5B,IAAI,CAACC,OAAO,MAAM,IAAId,WAAW,iBAAiB,CAAC,uBAAuB,EAAEa,KAAK,cAAc,EAAEE,OAAOC,IAAI,CAACV,UAAUW,IAAI,CAAC,OAAO;IAEnI,yFAAyF;IACzF,2FAA2F;IAC3F,2FAA2F;IAC3F,6DAA6D;IAC7D,IAAInB,gBAAgBc,QAAQ,CAACE,MAAMN,YAAY,CAACU,GAAG,CAAC,YAAY;QAC9D,MAAM,IAAIlB,WAAW,4BAA4B,CAAC,OAAO,EAAEa,KAAK,6LAA6L,CAAC;IAChQ;IAEA,OAAOC,MAAML,IAAI,CAACG;AACpB;AAEA,SAASO,OAAO,EAAEC,OAAO,QAAQ,YAAY;AAE7C,SAASC,UAAU,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,cAAc,QAAQ,mBAAmB"}

package/dist/esm/store/meta.d.ts

@@ -0,0 +1,3 @@
+import type { Store } from './types.js';
+export declare function getMeta(store: Store, key: string): Promise<string | null>;
+export declare function setMeta(store: Store, key: string, value: string | null): Promise<void>;

package/dist/esm/store/meta.js

@@ -0,0 +1,17 @@
+// Portable meta key/value reads for external callers (watch.ts, cli/status.ts) coarse enough
+// to cross the async boundary per call; reconcile/open's own hot-path meta access stays
+// inside store/shared.ts.
+export async function getMeta(store, key) {
+    const stmt = await store.prepare('SELECT value FROM meta WHERE key = ?');
+    const row = await stmt.get(key);
+    return row ? row.value : null;
+}
+export async function setMeta(store, key, value) {
+    if (value === null) {
+        const stmt = await store.prepare('DELETE FROM meta WHERE key = ?');
+        await stmt.run(key);
+        return;
+    }
+    const stmt = await store.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
+    await stmt.run(key, value);
+}

package/dist/esm/store/meta.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/meta.ts"],"sourcesContent":["import type { Store } from './types.ts';\n\n// Portable meta key/value reads for external callers (watch.ts, cli/status.ts) coarse enough\n// to cross the async boundary per call; reconcile/open's own hot-path meta access stays\n// inside store/shared.ts.\n\nexport async function getMeta(store: Store, key: string): Promise<string | null> {\n  const stmt = await store.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(store: Store, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await store.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await store.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n"],"names":["getMeta","store","key","stmt","prepare","row","get","value","setMeta","run"],"mappings":"AAEA,6FAA6F;AAC7F,wFAAwF;AACxF,0BAA0B;AAE1B,OAAO,eAAeA,QAAQC,KAAY,EAAEC,GAAW;IACrD,MAAMC,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMC,MAAO,MAAMF,KAAKG,GAAG,CAACJ;IAC5B,OAAOG,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQP,KAAY,EAAEC,GAAW,EAAEK,KAAoB;IAC3E,IAAIA,UAAU,MAAM;QAClB,MAAMJ,OAAO,MAAMF,MAAMG,OAAO,CAAC;QACjC,MAAMD,KAAKM,GAAG,CAACP;QACf;IACF;IACA,MAAMC,OAAO,MAAMF,MAAMG,OAAO,CAAC;IACjC,MAAMD,KAAKM,GAAG,CAACP,KAAKK;AACtB"}

package/dist/esm/store/shared.d.ts

@@ -0,0 +1,5 @@
+import type { Connection } from './types.js';
+export declare function quoteIdent(name: string): string;
+export declare function getColumns(conn: Connection): Promise<Set<string>>;
+export declare function getMeta(conn: Connection, key: string): Promise<string | null>;
+export declare function setMeta(conn: Connection, key: string, value: string | null): Promise<void>;

package/dist/esm/store/shared.js

@@ -0,0 +1,24 @@
+// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both
+// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).
+export function quoteIdent(name) {
+    return `"${name.split('"').join('""')}"`;
+}
+export async function getColumns(conn) {
+    const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');
+    const rows = await stmt.all();
+    return new Set(rows.map((r)=>r.name));
+}
+export async function getMeta(conn, key) {
+    const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');
+    const row = await stmt.get(key);
+    return row ? row.value : null;
+}
+export async function setMeta(conn, key, value) {
+    if (value === null) {
+        const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');
+        await stmt.run(key);
+        return;
+    }
+    const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');
+    await stmt.run(key, value);
+}

package/dist/esm/store/shared.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/shared.ts"],"sourcesContent":["import type { Connection } from './types.ts';\n\n// SQL and meta-table primitives both stores' open()/reconcile() need. Engine-neutral: both\n// stores' Connection satisfies the same async exec/prepare/runBatch shape (types.ts).\n\nexport function quoteIdent(name: string): string {\n  return `\"${name.split('\"').join('\"\"')}\"`;\n}\n\nexport async function getColumns(conn: Connection): Promise<Set<string>> {\n  const stmt = await conn.prepare('PRAGMA table_info(frontmatter)');\n  const rows = (await stmt.all()) as Array<{ name: string }>;\n  return new Set(rows.map((r) => r.name));\n}\n\nexport async function getMeta(conn: Connection, key: string): Promise<string | null> {\n  const stmt = await conn.prepare('SELECT value FROM meta WHERE key = ?');\n  const row = (await stmt.get(key)) as { value: string } | undefined;\n  return row ? row.value : null;\n}\n\nexport async function setMeta(conn: Connection, key: string, value: string | null): Promise<void> {\n  if (value === null) {\n    const stmt = await conn.prepare('DELETE FROM meta WHERE key = ?');\n    await stmt.run(key);\n    return;\n  }\n  const stmt = await conn.prepare('INSERT INTO meta (key, value) VALUES (?, ?) ON CONFLICT(key) DO UPDATE SET value = excluded.value');\n  await stmt.run(key, value);\n}\n"],"names":["quoteIdent","name","split","join","getColumns","conn","stmt","prepare","rows","all","Set","map","r","getMeta","key","row","get","value","setMeta","run"],"mappings":"AAEA,2FAA2F;AAC3F,sFAAsF;AAEtF,OAAO,SAASA,WAAWC,IAAY;IACrC,OAAO,CAAC,CAAC,EAAEA,KAAKC,KAAK,CAAC,KAAKC,IAAI,CAAC,MAAM,CAAC,CAAC;AAC1C;AAEA,OAAO,eAAeC,WAAWC,IAAgB;IAC/C,MAAMC,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMC,OAAQ,MAAMF,KAAKG,GAAG;IAC5B,OAAO,IAAIC,IAAIF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;AACvC;AAEA,OAAO,eAAeY,QAAQR,IAAgB,EAAES,GAAW;IACzD,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMQ,MAAO,MAAMT,KAAKU,GAAG,CAACF;IAC5B,OAAOC,MAAMA,IAAIE,KAAK,GAAG;AAC3B;AAEA,OAAO,eAAeC,QAAQb,IAAgB,EAAES,GAAW,EAAEG,KAAoB;IAC/E,IAAIA,UAAU,MAAM;QAClB,MAAMX,OAAO,MAAMD,KAAKE,OAAO,CAAC;QAChC,MAAMD,KAAKa,GAAG,CAACL;QACf;IACF;IACA,MAAMR,OAAO,MAAMD,KAAKE,OAAO,CAAC;IAChC,MAAMD,KAAKa,GAAG,CAACL,KAAKG;AACtB"}

package/dist/esm/store/sql-functions.d.ts

@@ -0,0 +1,2 @@
+export declare function hasImpl(field: unknown, value: unknown): number;
+export declare function basenameImpl(path: unknown, suffix?: unknown): string | null;

package/dist/esm/store/sql-functions.js

@@ -0,0 +1,29 @@
+import posix from 'node:path/posix';
+// has(field, value) / basename(path, suffix): the two sense-supplied SQL functions with no
+// engine-specific pieces, shared by sqlite/sql-functions.ts and duckdb/sql-functions.ts so each
+// store registers the identical implementation instead of a hand-copied one (DRY).
+// JSON-array field -> membership, string field -> substring, NULL -> false.
+export function hasImpl(field, value) {
+    if (field === null || field === undefined) return 0;
+    const needle = String(value);
+    if (typeof field === 'string') {
+        if (field.startsWith('[')) {
+            try {
+                const parsed = JSON.parse(field);
+                if (Array.isArray(parsed)) return parsed.some((item)=>String(item) === needle) ? 1 : 0;
+            } catch  {}
+        }
+        return field.includes(needle) ? 1 : 0;
+    }
+    return String(field).includes(needle) ? 1 : 0;
+}
+// Unix basename(path, suffix?): the filename, minus suffix when it ends with one. Neither
+// engine has a filename function of its own, and LIKE tricks that fake one also match folder
+// names.
+export function basenameImpl(path, suffix) {
+    if (path === null || path === undefined) return null;
+    const name = posix.basename(String(path));
+    const tail = suffix === null || suffix === undefined ? '' : String(suffix);
+    // POSIX: a suffix identical to the whole name is not removed (node's own basename strips it).
+    return tail !== '' && tail !== name && name.endsWith(tail) ? name.slice(0, -tail.length) : name;
+}