sensemaking 0.18.3 → 0.19.0

package/dist/esm/scan/reparse.js

@@ -3,25 +3,15 @@ import { availableParallelism } from 'node:os';
 import { dirname, join } from 'node:path';
 import { parseFile } from './index.js';
 import { reviveError } from './worker-error.js';
-// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house
-// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below
-// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.
+// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.
+// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.
 const _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;
-// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160
-// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph
-// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches
-// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends
-// worse (main-thread + OS contention on the 6 non-parse-worker cores).
+// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files
+// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.
 const DEFAULT_THRESHOLD = 200;
 const DEFAULT_MAX_WORKERS = 8;
-// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,
-// install-optional, node-version-install, node-exec-path all carry this same rule). A
-// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader
-// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs
-// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite
-// runs against src/ (which already requires a build for the exports specs).
-// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and
-// must not pay for this, let alone fail on it.
+// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited
+// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.
 let workerFile;
 function resolveWorkerFile() {
     if (workerFile) return workerFile;
@@ -40,9 +30,8 @@ function resolveWorkerFile() {
     }
     throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');
 }
-// Per-file feature filtering, shared by the serial loop (which reads `features` straight from
-// the caller, real registry or a test double) and the worker path (which can only pass `cfg`
-// across the thread boundary, so it derives `features` from activeFeatures(cfg) itself).
+// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and
+// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).
 export function featuresForFile(features, cfg, file) {
     return features.filter((feature)=>!feature.enabledForFile || feature.enabledForFile(cfg, file));
 }
@@ -56,15 +45,12 @@ function reparseSerial(files, features, cfg, onParsed) {
     }
     return results;
 }
-// Never the tree: a worker task carries one FileStat and returns only what parseFile already
-// returns -- extracted text and feature values, never an mdast tree. The worker reads the file
-// itself. Pool created once here and destroyed once dispatch finishes; never per task.
+// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --
+// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.
 async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
     const { Tinypool } = _require('tinypool');
-    // cfg and the caller's feature selection are constant for the whole dispatch, so they cross
-    // once per worker as workerData rather than once per file in the task payload. Features carry
-    // closures and cannot cross at all; their names can, and the worker resolves them back
-    // against the same registry, so a caller passing a subset gets that subset in both modes.
+    // cfg and the feature selection are constant for the dispatch, so they cross once per worker
+    // as workerData. Features carry closures and cannot cross at all; only their names do.
     const workerData = {
         cfg,
         featureNames: features.map((feature)=>feature.name)
@@ -77,11 +63,8 @@ async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
     });
     let done = 0;
     try {
-        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files`
-        // order regardless of which task finishes first, which is what first-seen column order
-        // and warning order depend on downstream. Progress ticks fire in completion order instead
-        // (only the count matters for a progress bar) -- free, since it rides the same per-task
-        // response tinypool already sends, no extra ping.
+        // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order
+        // regardless of task completion order, which first-seen column order depends on downstream.
         const results = await Promise.all(files.map(async (file)=>{
             const result = await pool.run(file);
             if (!result.ok) throw reviveError(result.error);
@@ -94,9 +77,8 @@ async function reparsePooled(files, features, cfg, onParsed, maxWorkers) {
         await pool.destroy();
         return results;
     } catch (err) {
-        // Tear the pool down before rethrowing, but never let a destroy failure replace the parse
-        // failure that caused it: that one names the file, this one names nothing. Not a `finally`
-        // for exactly that reason.
+        // Tear the pool down before rethrowing, but a destroy failure must not mask the parse
+        // failure that caused it -- not a `finally`, for that reason.
         await pool.destroy().catch(()=>{});
         throw err;
     }

package/dist/esm/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house\n// deferral reaches it. Deferred (not a top-level import) because most reconciles stay below\n// DEFAULT_THRESHOLD and should not pay for constructing pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving by\n// construction (one pass over `files`, pushed in order) so a future concurrent dispatch can\n// replace the loop body without changing this contract or either call site.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order across `files` -- the order\n  // callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-logical-core Apple M4 Pro: serial and pooled cross between 160\n// and 180 files (pool fixed cost here is ~85-100ms, not the ~300ms a heavier module graph\n// costs elsewhere), so 200 keeps a safety margin without giving up real wins. 8 workers matches\n// an interleaved worker-count sweep at 6,566 files -- 8 and 10 tie within noise, 12+ trends\n// worse (main-thread + OS contention on the 6 non-parse-worker cores).\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST always load from dist/cjs/ (house pattern: install-module-linked,\n// install-optional, node-version-install, node-exec-path all carry this same rule). A\n// worker_threads thread is a fresh realm and does not inherit the main thread's TS loader\n// hook, so a source path cannot work on any platform; dist/cjs loads from the esm and cjs\n// builds alike. Root is found the way cli.ts finds it, so this also resolves when the suite\n// runs against src/ (which already requires a build for the exports specs).\n// Resolved on first pooled dispatch, not at import: a tree under the threshold never pools and\n// must not pay for this, let alone fail on it.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (which reads `features` straight from\n// the caller, real registry or a test double) and the worker path (which can only pass `cfg`\n// across the thread boundary, so it derives `features` from activeFeatures(cfg) itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile already\n// returns -- extracted text and feature values, never an mdast tree. The worker reads the file\n// itself. Pool created once here and destroyed once dispatch finishes; never per task.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the caller's feature selection are constant for the whole dispatch, so they cross\n  // once per worker as workerData rather than once per file in the task payload. Features carry\n  // closures and cannot cross at all; their names can, and the worker resolves them back\n  // against the same registry, so a caller passing a subset gets that subset in both modes.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files`\n    // order regardless of which task finishes first, which is what first-seen column order\n    // and warning order depend on downstream. Progress ticks fire in completion order instead\n    // (only the count matters for a progress bar) -- free, since it rides the same per-task\n    // response tinypool already sends, no extra ping.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but never let a destroy failure replace the parse\n    // failure that caused it: that one names the file, this one names nothing. Not a `finally`\n    // for exactly that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["createRequire","availableParallelism","dirname","join","parseFile","reviveError","_require","require","url","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","Tinypool","workerData","featureNames","map","pool","filename","minThreads","maxThreads","Promise","all","result","run","ok","error","doc","warnings","destroy","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","newColumns","seen","Set","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,oBAAoB,QAAQ,UAAU;AAC/C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAQ1C,SAASC,SAAS,QAAQ,aAAa;AAEvC,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,yFAAyF;AACzF,4FAA4F;AAC5F,uFAAuF;AACvF,MAAMC,WAAW,OAAOC,YAAY,cAAcP,cAAc,YAAYQ,GAAG,IAAID;AAqBnF,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,4FAA4F;AAC5F,uEAAuE;AACvE,MAAME,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,gFAAgF;AAChF,sFAAsF;AACtF,0FAA0F;AAC1F,0FAA0F;AAC1F,4FAA4F;AAC5F,4EAA4E;AAC5E,+FAA+F;AAC/F,+CAA+C;AAC/C,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOb,cAAc,YAAYQ,GAAG;IAC1C,KAAK,MAAMM,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaR,KAAKD,QAAQW,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,8FAA8F;AAC9F,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAAC3B,UAAUiB,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,uFAAuF;AACvF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB;IAClJ,MAAM,EAAEC,QAAQ,EAAE,GAAG5B,SAAS;IAC9B,4FAA4F;IAC5F,8FAA8F;IAC9F,uFAAuF;IACvF,0FAA0F;IAC1F,MAAM6B,aAA8B;QAAEf;QAAKgB,cAAcjB,SAASkB,GAAG,CAAC,CAACd,UAAYA,QAAQR,IAAI;IAAE;IACjG,MAAMuB,OAAO,IAAIJ,SAAS;QAAEK,UAAU3B;QAAqB4B,YAAYP;QAAYQ,YAAYR;QAAYE;IAAW;IACtH,IAAIN,OAAO;IACX,IAAI;QACF,oFAAoF;QACpF,uFAAuF;QACvF,0FAA0F;QAC1F,wFAAwF;QACxF,kDAAkD;QAClD,MAAMD,UAAU,MAAMc,QAAQC,GAAG,CAC/BjB,MAAMW,GAAG,CAAC,OAAOhB;YACf,MAAMuB,SAAU,MAAMN,KAAKO,GAAG,CAACxB;YAC/B,IAAI,CAACuB,OAAOE,EAAE,EAAE,MAAMzC,YAAYuC,OAAOG,KAAK;YAC9CpB,qBAAAA,+BAAAA,SAAW,EAAEE;YACb,OAAO;gBAAEmB,KAAKJ,OAAOI,GAAG;gBAAEC,UAAUL,OAAOK,QAAQ;YAAC;QACtD;QAEF,MAAMX,KAAKY,OAAO;QAClB,OAAOtB;IACT,EAAE,OAAOuB,KAAK;QACZ,0FAA0F;QAC1F,2FAA2F;QAC3F,2BAA2B;QAC3B,MAAMb,KAAKY,OAAO,GAAGE,KAAK,CAAC,KAAO;QAClC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAa3B,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEkC,YAAiC,EAAE3B,QAAiC,EAAE4B,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;IACvC,MAAMwB,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAChD,qBAAqBT;IACvE,MAAM2B,UAAUF,MAAMiC,MAAM,IAAIH,YAAY,MAAMxB,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,cAAcR,cAAcC,OAAOP,UAAUC,KAAKO;IAElJ,MAAMiC,OAAoB,EAAE;IAC5B,MAAMX,WAAqB,EAAE;IAC7B,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,KAAK,MAAM,EAAEN,GAAG,EAAEC,UAAUe,YAAY,EAAE,IAAIpC,QAAS;QACrDqB,SAASlB,IAAI,IAAIiC;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACnB,IAAIoB,IAAI,EAAG;YACvC,IAAI,CAACN,KAAKO,GAAG,CAACJ,MAAM;gBAClBH,KAAKQ,GAAG,CAACL;gBACTJ,WAAW9B,IAAI,CAACkC;YAClB;QACF;QACAL,KAAK7B,IAAI,CAACiB;IACZ;IAEA,OAAO;QAAEY;QAAMX;QAAUY;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { availableParallelism } from 'node:os';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\n// Deferred (not top-level) because most reconciles stay below DEFAULT_THRESHOLD and shouldn't pay for pool machinery they never use.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\ntype FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n// extracted text and feature values. Pool created once here, destroyed once dispatch finishes.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n  const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n  // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n  // as workerData. Features carry closures and cannot cross at all; only their names do.\n  const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n  const pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n  let done = 0;\n  try {\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    const results = await Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n    await pool.destroy();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a destroy failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await pool.destroy().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["createRequire","availableParallelism","dirname","join","parseFile","reviveError","_require","require","url","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","Tinypool","workerData","featureNames","map","pool","filename","minThreads","maxThreads","Promise","all","result","run","ok","error","doc","warnings","destroy","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","newColumns","seen","Set","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,oBAAoB,QAAQ,UAAU;AAC/C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAQ1C,SAASC,SAAS,QAAQ,aAAa;AAEvC,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,qIAAqI;AACrI,MAAMC,WAAW,OAAOC,YAAY,cAAcP,cAAc,YAAYQ,GAAG,IAAID;AAmBnF,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAME,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIC;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOb,cAAc,YAAYQ,GAAG;IAC1C,KAAK,MAAMM,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaR,KAAKD,QAAQW,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAAC3B,UAAUiB,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,+FAA+F;AAC/F,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB;IAClJ,MAAM,EAAEC,QAAQ,EAAE,GAAG5B,SAAS;IAC9B,6FAA6F;IAC7F,uFAAuF;IACvF,MAAM6B,aAA8B;QAAEf;QAAKgB,cAAcjB,SAASkB,GAAG,CAAC,CAACd,UAAYA,QAAQR,IAAI;IAAE;IACjG,MAAMuB,OAAO,IAAIJ,SAAS;QAAEK,UAAU3B;QAAqB4B,YAAYP;QAAYQ,YAAYR;QAAYE;IAAW;IACtH,IAAIN,OAAO;IACX,IAAI;QACF,0FAA0F;QAC1F,4FAA4F;QAC5F,MAAMD,UAAU,MAAMc,QAAQC,GAAG,CAC/BjB,MAAMW,GAAG,CAAC,OAAOhB;YACf,MAAMuB,SAAU,MAAMN,KAAKO,GAAG,CAACxB;YAC/B,IAAI,CAACuB,OAAOE,EAAE,EAAE,MAAMzC,YAAYuC,OAAOG,KAAK;YAC9CpB,qBAAAA,+BAAAA,SAAW,EAAEE;YACb,OAAO;gBAAEmB,KAAKJ,OAAOI,GAAG;gBAAEC,UAAUL,OAAOK,QAAQ;YAAC;QACtD;QAEF,MAAMX,KAAKY,OAAO;QAClB,OAAOtB;IACT,EAAE,OAAOuB,KAAK;QACZ,sFAAsF;QACtF,8DAA8D;QAC9D,MAAMb,KAAKY,OAAO,GAAGE,KAAK,CAAC,KAAO;QAClC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAa3B,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEkC,YAAiC,EAAE3B,QAAiC,EAAE4B,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB9C;IACvC,MAAMwB,cAAasB,sBAAAA,QAAQtB,UAAU,cAAlBsB,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAChD,qBAAqBT;IACvE,MAAM2B,UAAUF,MAAMiC,MAAM,IAAIH,YAAY,MAAMxB,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,cAAcR,cAAcC,OAAOP,UAAUC,KAAKO;IAElJ,MAAMiC,OAAoB,EAAE;IAC5B,MAAMX,WAAqB,EAAE;IAC7B,MAAMY,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIT;IACrB,KAAK,MAAM,EAAEN,GAAG,EAAEC,UAAUe,YAAY,EAAE,IAAIpC,QAAS;QACrDqB,SAASlB,IAAI,IAAIiC;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACnB,IAAIoB,IAAI,EAAG;YACvC,IAAI,CAACN,KAAKO,GAAG,CAACJ,MAAM;gBAClBH,KAAKQ,GAAG,CAACL;gBACTJ,WAAW9B,IAAI,CAACkC;YAClB;QACF;QACAL,KAAK7B,IAAI,CAACiB;IACZ;IAEA,OAAO;QAAEY;QAAMX;QAAUY;IAAW;AACtC"}

package/dist/esm/scan/worker-error.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/worker-error.ts"],"sourcesContent":["import { SenseError, type SenseErrorCode } from '../errors.ts';\n\n// Structured clone across a worker_threads boundary drops a subclass and its custom\n// properties, so a SenseError arrives as a plain Error with `code` undefined -- and `code`\n// drives caller behavior. The worker serializes explicitly (computed on its side, where\n// `instanceof SenseError` is still meaningful) and the main thread rebuilds by hand.\nexport interface WorkerErrorPayload {\n  name: string;\n  code?: SenseErrorCode;\n  message: string;\n  stack?: string;\n}\n\nexport function serializeError(err: unknown): WorkerErrorPayload {\n  if (err instanceof Error) return { name: err.name, code: err instanceof SenseError ? err.code : undefined, message: err.message, stack: err.stack };\n  return { name: 'Error', message: String(err) };\n}\n\nexport function reviveError(payload: WorkerErrorPayload): Error {\n  const err = payload.code ? new SenseError(payload.code, payload.message) : new Error(payload.message);\n  err.name = payload.name;\n  if (payload.stack) err.stack = payload.stack;\n  return err;\n}\n"],"names":["SenseError","serializeError","err","Error","name","code","undefined","message","stack","String","reviveError","payload"],"mappings":"AAAA,SAASA,UAAU,QAA6B,eAAe;AAa/D,OAAO,SAASC,eAAeC,GAAY;IACzC,IAAIA,eAAeC,OAAO,OAAO;QAAEC,MAAMF,IAAIE,IAAI;QAAEC,MAAMH,eAAeF,aAAaE,IAAIG,IAAI,GAAGC;QAAWC,SAASL,IAAIK,OAAO;QAAEC,OAAON,IAAIM,KAAK;IAAC;IAClJ,OAAO;QAAEJ,MAAM;QAASG,SAASE,OAAOP;IAAK;AAC/C;AAEA,OAAO,SAASQ,YAAYC,OAA2B;IACrD,MAAMT,MAAMS,QAAQN,IAAI,GAAG,IAAIL,WAAWW,QAAQN,IAAI,EAAEM,QAAQJ,OAAO,IAAI,IAAIJ,MAAMQ,QAAQJ,OAAO;IACpGL,IAAIE,IAAI,GAAGO,QAAQP,IAAI;IACvB,IAAIO,QAAQH,KAAK,EAAEN,IAAIM,KAAK,GAAGG,QAAQH,KAAK;IAC5C,OAAON;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/worker-error.ts"],"sourcesContent":["import { SenseError, type SenseErrorCode } from '../errors.ts';\n\n// Structured clone across a worker_threads boundary drops a subclass and its custom properties, so\n// a SenseError arrives as a plain Error with `code` undefined (and `code` drives caller behavior): the worker serializes explicitly and the main thread rebuilds by hand.\nexport interface WorkerErrorPayload {\n  name: string;\n  code?: SenseErrorCode;\n  message: string;\n  stack?: string;\n}\n\nexport function serializeError(err: unknown): WorkerErrorPayload {\n  if (err instanceof Error) return { name: err.name, code: err instanceof SenseError ? err.code : undefined, message: err.message, stack: err.stack };\n  return { name: 'Error', message: String(err) };\n}\n\nexport function reviveError(payload: WorkerErrorPayload): Error {\n  const err = payload.code ? new SenseError(payload.code, payload.message) : new Error(payload.message);\n  err.name = payload.name;\n  if (payload.stack) err.stack = payload.stack;\n  return err;\n}\n"],"names":["SenseError","serializeError","err","Error","name","code","undefined","message","stack","String","reviveError","payload"],"mappings":"AAAA,SAASA,UAAU,QAA6B,eAAe;AAW/D,OAAO,SAASC,eAAeC,GAAY;IACzC,IAAIA,eAAeC,OAAO,OAAO;QAAEC,MAAMF,IAAIE,IAAI;QAAEC,MAAMH,eAAeF,aAAaE,IAAIG,IAAI,GAAGC;QAAWC,SAASL,IAAIK,OAAO;QAAEC,OAAON,IAAIM,KAAK;IAAC;IAClJ,OAAO;QAAEJ,MAAM;QAASG,SAASE,OAAOP;IAAK;AAC/C;AAEA,OAAO,SAASQ,YAAYC,OAA2B;IACrD,MAAMT,MAAMS,QAAQN,IAAI,GAAG,IAAIL,WAAWW,QAAQN,IAAI,EAAEM,QAAQJ,OAAO,IAAI,IAAIJ,MAAMQ,QAAQJ,OAAO;IACpGL,IAAIE,IAAI,GAAGO,QAAQP,IAAI;IACvB,IAAIO,QAAQH,KAAK,EAAEN,IAAIM,KAAK,GAAGG,QAAQH,KAAK;IAC5C,OAAON;AACT"}

package/dist/esm/store/cache.d.ts

@@ -0,0 +1,2 @@
+import type { ResolvedConfig } from '../config/index.js';
+export declare function clearCache(cfg: ResolvedConfig): void;

package/dist/esm/store/cache.js

@@ -0,0 +1,11 @@
+import { rmSync } from 'node:fs';
+import { join } from 'node:path';
+import { STATE_DIR } from '../config/index.js';
+// Deletes the cache directory and nothing else: the next open() reconciles anyway. Engine-neutral,
+// since every store keeps its files under the same STATE_DIR.
+export function clearCache(cfg) {
+    rmSync(join(cfg.baseDir, STATE_DIR), {
+        recursive: true,
+        force: true
+    });
+}

package/dist/esm/store/cache.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/cache.ts"],"sourcesContent":["import { rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { ResolvedConfig } from '../config/index.ts';\nimport { STATE_DIR } from '../config/index.ts';\n\n// Deletes the cache directory and nothing else: the next open() reconciles anyway. Engine-neutral,\n// since every store keeps its files under the same STATE_DIR.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["rmSync","join","STATE_DIR","clearCache","cfg","baseDir","recursive","force"],"mappings":"AAAA,SAASA,MAAM,QAAQ,UAAU;AACjC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,mGAAmG;AACnG,8DAA8D;AAC9D,OAAO,SAASC,WAAWC,GAAmB;IAC5CJ,OAAOC,KAAKG,IAAIC,OAAO,EAAEH,YAAY;QAAEI,WAAW;QAAMC,OAAO;IAAK;AACtE"}

package/dist/esm/store/duckdb/batch.js

@@ -1,16 +1,5 @@
-// Rewrites a single-row parameterized statement -- the shape every write loop in this codebase
-// prepares once and calls with N different param sets -- into one statement DuckDB executes over
-// all N rows in a single round trip. A columnar engine's cost is row-at-a-time writes themselves,
-// not just the async crossing to reach them, so this recognizes the three statement shapes the
-// store issues (INSERT ... VALUES (...), UPDATE ... SET ... WHERE ..., DELETE ... WHERE ...) and
-// rewrites each into a multi-row form, verified against a live DuckDB connection at
-// implementation time (PRINCIPLES: documented-means-tested). Unrecognized SQL returns null;
-// connection.ts's runBatch falls back to a prepare-once, bind-and-run loop -- still one call
-// from outside the store.
-//
-// This only ever runs against sense's own SQL literals (reconcile, the feature hooks), never
-// arbitrary user SQL (`sense sql` is a separate, unrewritten passthrough), so the shapes below
-// are exactly what those call sites emit, not a general parser.
+// Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --
+// DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).
 const INSERT_RE = /^(INSERT\s+(?:OR\s+\w+\s+)?INTO\s+\S+\s*\([^)]*\)\s*VALUES\s*)\(([^()]*)\)(.*)$/is;
 const UPDATE_RE = /^UPDATE\s+(\S+)\s+SET\s+(.+?)\s+WHERE\s+(.+)$/is;
 const DELETE_RE = /^DELETE\s+FROM\s+(\S+)\s+WHERE\s+(.+)$/is;
@@ -18,9 +7,8 @@ function placeholderCount(tuple) {
     var _tuple_match;
     return ((_tuple_match = tuple.match(/\?/g)) !== null && _tuple_match !== void 0 ? _tuple_match : []).length;
 }
-// "a = ?, b = ?" or "a = ? AND b = ?" -> ["a", "b"]. Only ever called on clauses already
-// shaped that way by the regexes above; a clause that doesn't fit yields a column list whose
-// length won't match the caller's row width, which rewriteUpdate/rewriteDelete then reject.
+// "a = ?, b = ?" or "a = ? AND b = ?" -> ["a", "b"]. A clause that doesn't fit yields a shorter
+// list, which rewriteUpdate/rewriteDelete reject as a width mismatch.
 function equalityColumns(clause, sep) {
     const parts = clause.split(sep).map((p)=>p.trim());
     const cols = [];

package/dist/esm/store/duckdb/batch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites a single-row parameterized statement -- the shape every write loop in this codebase\n// prepares once and calls with N different param sets -- into one statement DuckDB executes over\n// all N rows in a single round trip. A columnar engine's cost is row-at-a-time writes themselves,\n// not just the async crossing to reach them, so this recognizes the three statement shapes the\n// store issues (INSERT ... VALUES (...), UPDATE ... SET ... WHERE ..., DELETE ... WHERE ...) and\n// rewrites each into a multi-row form, verified against a live DuckDB connection at\n// implementation time (PRINCIPLES: documented-means-tested). Unrecognized SQL returns null;\n// connection.ts's runBatch falls back to a prepare-once, bind-and-run loop -- still one call\n// from outside the store.\n//\n// This only ever runs against sense's own SQL literals (reconcile, the feature hooks), never\n// arbitrary user SQL (`sense sql` is a separate, unrewritten passthrough), so the shapes below\n// are exactly what those call sites emit, not a general parser.\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. Only ever called on clauses already\n// shaped that way by the regexes above; a clause that doesn't fit yields a column list whose\n// length won't match the caller's row width, which rewriteUpdate/rewriteDelete then reject.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["INSERT_RE","UPDATE_RE","DELETE_RE","placeholderCount","tuple","match","length","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","join","rewriteInsert","sql","head","tail","rows","rewriteUpdate","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,kGAAkG;AAClG,+FAA+F;AAC/F,iGAAiG;AACjG,oFAAoF;AACpF,4FAA4F;AAC5F,6FAA6F;AAC7F,0BAA0B;AAC1B,EAAE;AACF,6FAA6F;AAC7F,+FAA+F;AAC/F,gEAAgE;AAEhE,MAAMA,YAAY;AAClB,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,4FAA4F;AAC5F,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEnB,QAAQgB;IAAM,GAAG,IAAM,KAAKI,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOF,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAME,KAAKG,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIjB,UAAUkB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGY,MAAMzB,OAAO0B,KAAK,GAAGb;IAC9B,MAAMK,QAAQnB,iBAAiBC;IAC/B,IAAIkB,UAAU,GAAG,OAAO;IACxB,MAAMS,OAAOP,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEjB,MAAM,CAAC,CAAC,EAAEsB,IAAI,CAAC;IACvE,OAAO;QAAEE,KAAK,GAAGC,OAAOE,OAAOD,MAAM;QAAER;IAAM;AAC/C;AAEA,OAAO,SAASU,cAAcJ,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIhB,UAAUiB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOC,WAAWC,YAAY,GAAGlB;IAC1C,MAAMmB,UAAU7B,gBAAgB2B,WAAW;IAC3C,MAAMG,YAAY9B,gBAAgB4B,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQxB,GAAG,CAAC,CAAC4B,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IAC9D,MAAMe,WAAWJ,UAAUzB,GAAG,CAAC,CAAC4B,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEK,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEnB,OAAOC,UAAUiB,SAAShC,MAAM,EAAE,UAAU,EAAEgC,SAASZ,IAAI,CAAC,MAAM,QAAQ,EAAEe,UAAU;QAAEnB,OAAOgB,SAAShC,MAAM;IAAC;AAChL;AAEA,OAAO,SAASoC,cAAcd,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIf,UAAUgB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOE,YAAY,GAAGlB;IAC/B,MAAMF,OAAOR,gBAAgB4B,aAAa;IAC1C,IAAI,CAACpB,MAAM,OAAO;IAClB,IAAIA,KAAKT,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEsB,KAAK,CAAC,YAAY,EAAEK,MAAM,OAAO,EAAElB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEnB,QAAQe;YAAS,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEJ,OAAO;QAAE;IACjI;IACA,OAAO;QAAEM,KAAK,CAAC,YAAY,EAAEK,MAAM,QAAQ,EAAElB,KAAKW,IAAI,CAAC,MAAM,aAAa,EAAEN,OAAOC,UAAUN,KAAKT,MAAM,EAAE,CAAC,CAAC;QAAEgB,OAAOP,KAAKT,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASqC,aAAaf,GAAW,EAAEP,QAAgB;QACjDM,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcC,KAAKP,uBAAnBM,4BAAAA,iBAAgCK,cAAcJ,KAAKP,uBAAnDM,kBAAAA,OAAgEe,cAAcd,KAAKP;AAC5F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites the store's single-row parameterized INSERT/UPDATE/DELETE into one multi-row statement --\n// DuckDB's columnar engine pays for row-at-a-time writes, not just the round trip (PRINCIPLES: documented-means-tested).\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. A clause that doesn't fit yields a shorter\n// list, which rewriteUpdate/rewriteDelete reject as a width mismatch.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["INSERT_RE","UPDATE_RE","DELETE_RE","placeholderCount","tuple","match","length","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","join","rewriteInsert","sql","head","tail","rows","rewriteUpdate","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,qGAAqG;AACrG,yHAAyH;AAEzH,MAAMA,YAAY;AAClB,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,gGAAgG;AAChG,sEAAsE;AACtE,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEnB,QAAQgB;IAAM,GAAG,IAAM,KAAKI,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOF,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAME,KAAKG,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIjB,UAAUkB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGY,MAAMzB,OAAO0B,KAAK,GAAGb;IAC9B,MAAMK,QAAQnB,iBAAiBC;IAC/B,IAAIkB,UAAU,GAAG,OAAO;IACxB,MAAMS,OAAOP,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEjB,MAAM,CAAC,CAAC,EAAEsB,IAAI,CAAC;IACvE,OAAO;QAAEE,KAAK,GAAGC,OAAOE,OAAOD,MAAM;QAAER;IAAM;AAC/C;AAEA,OAAO,SAASU,cAAcJ,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIhB,UAAUiB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOC,WAAWC,YAAY,GAAGlB;IAC1C,MAAMmB,UAAU7B,gBAAgB2B,WAAW;IAC3C,MAAMG,YAAY9B,gBAAgB4B,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQxB,GAAG,CAAC,CAAC4B,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IAC9D,MAAMe,WAAWJ,UAAUzB,GAAG,CAAC,CAAC4B,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEK,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEnB,OAAOC,UAAUiB,SAAShC,MAAM,EAAE,UAAU,EAAEgC,SAASZ,IAAI,CAAC,MAAM,QAAQ,EAAEe,UAAU;QAAEnB,OAAOgB,SAAShC,MAAM;IAAC;AAChL;AAEA,OAAO,SAASoC,cAAcd,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIf,UAAUgB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOE,YAAY,GAAGlB;IAC/B,MAAMF,OAAOR,gBAAgB4B,aAAa;IAC1C,IAAI,CAACpB,MAAM,OAAO;IAClB,IAAIA,KAAKT,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEsB,KAAK,CAAC,YAAY,EAAEK,MAAM,OAAO,EAAElB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEnB,QAAQe;YAAS,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEJ,OAAO;QAAE;IACjI;IACA,OAAO;QAAEM,KAAK,CAAC,YAAY,EAAEK,MAAM,QAAQ,EAAElB,KAAKW,IAAI,CAAC,MAAM,aAAa,EAAEN,OAAOC,UAAUN,KAAKT,MAAM,EAAE,CAAC,CAAC;QAAEgB,OAAOP,KAAKT,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASqC,aAAaf,GAAW,EAAEP,QAAgB;QACjDM,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcC,KAAKP,uBAAnBM,4BAAAA,iBAAgCK,cAAcJ,KAAKP,uBAAnDM,kBAAAA,OAAgEe,cAAcd,KAAKP;AAC5F"}

package/dist/esm/store/duckdb/connection.js

@@ -1,8 +1,7 @@
 import { withTransaction } from '../transaction.js';
 import { rewriteBatch } from './batch.js';
-// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's
-// small ints are numbers and consumers assume number. Beyond the safe range stays BigInt (the
-// setReadBigInts contract); stringifyJson keeps JSON output safe.
+// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small
+// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.
 const MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);
 export function storeValueToJs(value) {
     return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;
@@ -12,21 +11,13 @@ export function storeRowToJs(row) {
     for (const [key, value] of Object.entries(row))out[key] = storeValueToJs(value);
     return out;
 }
-// @duckdb/node-api tracks every prepared statement itself and destroys any survivor when the
-// connection closes, but sense's stores stay open for a whole `sense watch` session -- nothing
-// destroys one before then. The portable Statement interface has no dispose (sqlite's node:sqlite
-// statements need none, and callers like graph/traverse.ts's ring loop reuse one Statement across
-// several run/get/all calls, so a per-call destroy would be wrong), so cleanup happens here,
-// locally, once nothing references the wrapper any more. destroySync() is safe to call more than
-// once -- runBatch below and vectors.ts already call it explicitly while the same connection-close
-// sweep can also reach a statement that outlives its wrapper, with no reported issue.
+// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.
+// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.
 const preparedFinalizer = new FinalizationRegistry((prepared)=>{
     prepared.destroySync();
 });
-// Wraps one already-prepared DuckDB statement: prepare() is the only async step (Store.prepare
-// already crosses the async boundary once), so columns() reads back synchronously from it and
-// run/get/all rebind and re-execute the same native statement, matching how a caller (e.g.
-// graph/traverse.ts's ring loop) reuses one Statement across several calls.
+// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and
+// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.
 let DuckdbStatement = class DuckdbStatement {
     async run(...params) {
         if (params.length > 0) this.prepared.bind(params);
@@ -74,11 +65,8 @@ export function createConnection(duckdb) {
         async prepare (sql) {
             return new DuckdbStatement(await duckdb.prepare(sql));
         },
-        // One crossing regardless of row count: rewriteBatch turns the caller's single-row SQL into
-        // one multi-row statement when it recognizes the shape (see batch.ts); anything else falls
-        // back to a bind-and-run loop over one prepared statement, still a single call to the caller.
-        // Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so
-        // a standalone batch stays atomic.
+        // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),
+        // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.
         async runBatch (sql, paramRows) {
             if (paramRows.length === 0) return;
             await withTransaction(conn, async ()=>{

package/dist/esm/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's\n// small ints are numbers and consumers assume number. Beyond the safe range stays BigInt (the\n// setReadBigInts contract); stringifyJson keeps JSON output safe.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api tracks every prepared statement itself and destroys any survivor when the\n// connection closes, but sense's stores stay open for a whole `sense watch` session -- nothing\n// destroys one before then. The portable Statement interface has no dispose (sqlite's node:sqlite\n// statements need none, and callers like graph/traverse.ts's ring loop reuse one Statement across\n// several run/get/all calls, so a per-call destroy would be wrong), so cleanup happens here,\n// locally, once nothing references the wrapper any more. destroySync() is safe to call more than\n// once -- runBatch below and vectors.ts already call it explicitly while the same connection-close\n// sweep can also reach a statement that outlives its wrapper, with no reported issue.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step (Store.prepare\n// already crosses the async boundary once), so columns() reads back synchronously from it and\n// run/get/all rebind and re-execute the same native statement, matching how a caller (e.g.\n// graph/traverse.ts's ring loop) reuses one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch turns the caller's single-row SQL into\n    // one multi-row statement when it recognizes the shape (see batch.ts); anything else falls\n    // back to a bind-and-run loop over one prepared statement, still a single call to the caller.\n    // Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so\n    // a standalone batch stays atomic.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,0FAA0F;AAC1F,8FAA8F;AAC9F,kEAAkE;AAClE,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,mGAAmG;AACnG,sFAAsF;AACtF,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,+FAA+F;AAC/F,8FAA8F;AAC9F,2FAA2F;AAC3F,4EAA4E;AAC5E,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAEA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,4FAA4F;QAC5F,2FAA2F;QAC3F,8FAA8F;QAC9F,4FAA4F;QAC5F,mCAAmC;QACnC,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAEA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}

package/dist/esm/store/duckdb/fieldStats.js

@@ -1,22 +1,14 @@
 import { quoteIdent } from '../shared.js';
-// DuckDB's frontmatter columns are VARIANT (reconcile.ts's comment on why), so typeof() reports
-// VARIANT for every field regardless of the boxed value; variant_typeof() reports the boxed
-// native type instead (spike-verified against mapValue()'s actual bind shapes: a bound JS bigint
-// always boxes as INT128, a non-integer JS number always boxes as DOUBLE, a string as VARCHAR --
-// literal SQL values box narrower, e.g. INTEGER/DECIMAL(p, s), which this still covers since a
-// query issued through `sense sql` can produce either). Mapped onto the shared integer/real/text
-// vocabulary sqlite/turso's typeof() already emits natively; an unrecognized name throws rather
-// than being silently called text (no-silent-modes).
+// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native
+// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).
 function classifyVariantType(name, field) {
     if (/^U?INT\d+$/.test(name)) return 'integer';
     if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';
     if (name === 'VARCHAR') return 'text';
     throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
 }
-// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per
-// column, so the query stays O(columns) rather than O(rows x columns). '|' is the string_agg
-// separator, not ',': DECIMAL's own display name ("DECIMAL(4, 2)") contains a comma, which a
-// ',' separator could not be split back apart from.
+// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).
+// '|' is the separator, not ',': DECIMAL's display name ("DECIMAL(4, 2)") contains a comma.
 export async function fieldStats(conn, columns, scopeWhere) {
     const exprs = columns.map((name, i)=>{
         const quoted = quoteIdent(name);

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts's comment on why), so typeof() reports\n// VARIANT for every field regardless of the boxed value; variant_typeof() reports the boxed\n// native type instead (spike-verified against mapValue()'s actual bind shapes: a bound JS bigint\n// always boxes as INT128, a non-integer JS number always boxes as DOUBLE, a string as VARCHAR --\n// literal SQL values box narrower, e.g. INTEGER/DECIMAL(p, s), which this still covers since a\n// query issued through `sense sql` can produce either). Mapped onto the shared integer/real/text\n// vocabulary sqlite/turso's typeof() already emits natively; an unrecognized name throws rather\n// than being silently called text (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per\n// column, so the query stays O(columns) rather than O(rows x columns). '|' is the string_agg\n// separator, not ',': DECIMAL's own display name (\"DECIMAL(4, 2)\") contains a comma, which a\n// ',' separator could not be split back apart from.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","fieldStats","conn","columns","scopeWhere","exprs","map","i","quoted","stmt","prepare","join","row","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,gGAAgG;AAChG,4FAA4F;AAC5F,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,iGAAiG;AACjG,gGAAgG;AAChG,qDAAqD;AACrD,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,6FAA6F;AAC7F,oDAAoD;AACpD,OAAO,eAAeI,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACV,MAAMW;QAC/B,MAAMC,SAASd,WAAWE;QAC1B,OAAO,CAAC,MAAM,EAAEY,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IACvI;IACA,MAAME,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEP,YAAY;IAC3F,MAAMQ,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOV,QAAQG,GAAG,CAAC,CAACV,MAAMW;YACVK,OAEyBA;QAFvC,MAAME,MAAM,EAAEF,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAIG,KAAK,CAAC,KAAKC,MAAM,CAACC;QAC/D,MAAMC,QAAQ;eAAI,IAAIC,IAAIL,IAAIR,GAAG,CAAC,CAACc,IAAMzB,oBAAoByB,GAAGxB;SAAQ,CAACyB,IAAI;QAC7E,OAAO;YAAExB,OAAOD;YAAM0B,UAAUC,QAAOX,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;YAAIY,MAAMN,MAAMP,IAAI,CAAC;QAAK;IACnF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts), so typeof() always reports VARIANT; variant_typeof() reports the boxed native\n// type instead -- INT128, DOUBLE, VARCHAR, or a literal's narrower type like DECIMAL(p, s) -- mapped onto sqlite/turso's integer/real/text vocabulary; an unrecognized name throws (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per column keeps this O(columns), not O(rows x columns).\n// '|' is the separator, not ',': DECIMAL's display name (\"DECIMAL(4, 2)\") contains a comma.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","fieldStats","conn","columns","scopeWhere","exprs","map","i","quoted","stmt","prepare","join","row","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,yIAAyI;AACzI,sMAAsM;AACtM,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,8IAA8I;AAC9I,4FAA4F;AAC5F,OAAO,eAAeI,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACV,MAAMW;QAC/B,MAAMC,SAASd,WAAWE;QAC1B,OAAO,CAAC,MAAM,EAAEY,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IACvI;IACA,MAAME,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEP,YAAY;IAC3F,MAAMQ,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOV,QAAQG,GAAG,CAAC,CAACV,MAAMW;YACVK,OAEyBA;QAFvC,MAAME,MAAM,EAAEF,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAIG,KAAK,CAAC,KAAKC,MAAM,CAACC;QAC/D,MAAMC,QAAQ;eAAI,IAAIC,IAAIL,IAAIR,GAAG,CAAC,CAACc,IAAMzB,oBAAoByB,GAAGxB;SAAQ,CAACyB,IAAI;QAC7E,OAAO;YAAExB,OAAOD;YAAM0B,UAAUC,QAAOX,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;YAAIY,MAAMN,MAAMP,IAAI,CAAC;QAAK;IACnF;AACF"}

package/dist/esm/store/duckdb/lexical.js

@@ -1,26 +1,22 @@
 // D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /
 // unspaced scripts, JS excerpts for every doc (no snippet()).
 import { SenseError } from '../../errors.js';
-import { UNSPACED_SCRIPTS } from '../../text/segment.js';
+import { hasUnspacedRun } from '../../text/segment.js';
 import { withTransaction } from '../transaction.js';
 const FIELDS = [
     'title',
     'summary',
     'text'
 ];
-// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's
-// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored
-// separately and combined here.
+// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's match_bm25 has no
+// field_weights argument (verified 1.5.5), so each field is scored separately and combined here.
 const FIELD_WEIGHT = {
     title: 10,
     summary: 5,
     text: 1
 };
-const UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
-// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would
-// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).
-// Checked against terms with quoted spans blanked -- those go through contains() and are
-// supported.
+// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would otherwise silently
+// treat as literal terms (PRINCIPLES: no-silent-modes). Checked with quoted spans blanked; those go through contains() and are supported.
 const FTS5_OPERATORS = [
     {
         label: 'prefix query',
@@ -54,9 +50,8 @@ function unsupportedOperator(terms) {
     }
     return null;
 }
-// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index
-// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --
-// and any author-quoted phrase, any script -- go through contains() instead of the fts index.
+// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index the whole run as one token
+// (same gap FTS5 has without the `_seg` sidecar), so such runs -- and any author-quoted phrase, any script -- go through contains() instead.
 function splitTerms(terms) {
     const words = [];
     const substrings = [];
@@ -64,14 +59,14 @@ function splitTerms(terms) {
         const phrase = inner.trim();
         if (phrase.length === 0) return ' ';
         substrings.push(phrase);
-        if (!UNSPACED_RUN.test(phrase)) {
+        if (!hasUnspacedRun(phrase)) {
             for (const w of phrase.split(/\s+/))if (w.length > 0) words.push(w);
         }
         return ' ';
     });
     for (const tok of withoutPhrases.split(/\s+/)){
         if (tok.length === 0) continue;
-        if (UNSPACED_RUN.test(tok)) substrings.push(tok);
+        if (hasUnspacedRun(tok)) substrings.push(tok);
         else words.push(tok);
     }
     return {
@@ -79,55 +74,54 @@ function splitTerms(terms) {
         substrings
     };
 }
-// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this
-// pays the full rebuild -- but only once per store instance's first lexical query, and only
-// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).
+// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this pays the full rebuild -- but only
+// once per store instance's first lexical query, and only when content actually changed since (FtsIndexState / store.ts's markStale()).
 async function ensureFtsFresh(conn, state) {
     if (!state.stale) return;
     await conn.exec('INSTALL fts; LOAD fts;');
     await withTransaction(conn, async ()=>{
-        // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either
-        // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise
-        // silently drop common query words (e.g. "and") from the index but not from match_bm25's
-        // conjunctive gate, making a bare multi-word query that contains one match nothing.
+        // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either (verified 1.5.5); the fts extension's
+        // default 571-word list would otherwise drop common words (e.g. "and") from the index but not match_bm25's conjunctive gate, breaking multi-word matches.
         await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);
     });
     state.stale = false;
 }
-// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both
-// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are
-// emitted -- positional `?` binding requires that order to match the assembled SQL text.
+// Field-weighted fragments (title 10 / summary 5 / text 1), with params emitted in the exact left-to-right order the assembled SQL needs for positional `?` binding.
+// score/gate params stay in separate arrays until joined at the end, since the SQL puts every score part before every gate part; sharing one array would shift binds once a query has both words and substrings (PRINCIPLES: no-silent-modes).
 function buildScoreAndGate(words, substrings) {
     const scoreParts = [];
     const gateParts = [];
-    const params = [];
+    const scoreParams = [];
+    const gateParams = [];
     if (words.length > 0) {
         const wordQuery = words.join(' ');
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
-            params.push(wordQuery);
+            scoreParams.push(wordQuery);
         }
         gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
-        params.push(wordQuery);
+        gateParams.push(wordQuery);
     }
     for (const raw of substrings){
         const needle = raw.toLowerCase();
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
-            params.push(needle);
+            scoreParams.push(needle);
         }
         gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
-        params.push(needle);
+        gateParams.push(needle);
     }
     return {
         scoreSql: scoreParts.join(' + '),
         gateSql: gateParts.join(' AND '),
-        params
+        params: [
+            ...scoreParams,
+            ...gateParams
+        ]
     };
 }
-// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's
-// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes
-// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.
+// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's queryLexical takes).
+// `hit` is always NULL: no snippet() equivalent exists, so every row goes through the caller's JS excerpt fallback (commands/search.ts).
 export async function queryLexical(conn, terms, opts, state) {
     const unsupported = unsupportedOperator(terms);
     if (unsupported !== null) {
@@ -148,9 +142,8 @@ export async function queryLexical(conn, terms, opts, state) {
     const stmt = await conn.prepare(sql);
     return await stmt.all(...params, limit);
 }
-// One instance per store: `stale` starts true (a fresh connection cannot know whether an
-// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back
-// to true whenever content changes, so the next query rebuilds before ranking against it.
+// One instance per store: `stale` starts true (a fresh connection can't know whether an on-disk fts schema matches `content`),
+// and store.ts's reconcile wrapper flips it back to true whenever content changes, so the next query rebuilds first.
 export function createLexicalIndex(conn) {
     const state = {
         stale: true