sensemaking 0.18.2 → 0.18.4

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","SUPPORTED_CONFIG_VERSION","SenseError","printRows","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE9I,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIjE,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","SUPPORTED_CONFIG_VERSION","SenseError","printRows","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE9I,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAIjE,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/lib/atomic-write.d.ts

@@ -0,0 +1 @@
+export declare function writeFileAtomic(path: string, data: string | Buffer): void;

package/dist/esm/lib/atomic-write.js

@@ -0,0 +1,17 @@
+import { renameSync, unlinkSync, writeFileSync } from 'node:fs';
+// Stage-then-swap: write beside the target, then rename over it. A crash or full disk mid-write
+// leaves the target untouched instead of truncated. Matches embed/store.ts's model-download idiom.
+export function writeFileAtomic(path, data) {
+    const tmp = `${path}.part`;
+    writeFileSync(tmp, data);
+    try {
+        renameSync(tmp, path);
+    } catch (err) {
+        try {
+            unlinkSync(tmp);
+        } catch  {
+        // best-effort cleanup; the rename error below is what matters
+        }
+        throw err;
+    }
+}

package/dist/esm/lib/atomic-write.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/lib/atomic-write.ts"],"sourcesContent":["import { renameSync, unlinkSync, writeFileSync } from 'node:fs';\n\n// Stage-then-swap: write beside the target, then rename over it. A crash or full disk mid-write\n// leaves the target untouched instead of truncated. Matches embed/store.ts's model-download idiom.\nexport function writeFileAtomic(path: string, data: string | Buffer): void {\n  const tmp = `${path}.part`;\n  writeFileSync(tmp, data);\n  try {\n    renameSync(tmp, path);\n  } catch (err) {\n    try {\n      unlinkSync(tmp);\n    } catch {\n      // best-effort cleanup; the rename error below is what matters\n    }\n    throw err;\n  }\n}\n"],"names":["renameSync","unlinkSync","writeFileSync","writeFileAtomic","path","data","tmp","err"],"mappings":"AAAA,SAASA,UAAU,EAAEC,UAAU,EAAEC,aAAa,QAAQ,UAAU;AAEhE,gGAAgG;AAChG,mGAAmG;AACnG,OAAO,SAASC,gBAAgBC,IAAY,EAAEC,IAAqB;IACjE,MAAMC,MAAM,GAAGF,KAAK,KAAK,CAAC;IAC1BF,cAAcI,KAAKD;IACnB,IAAI;QACFL,WAAWM,KAAKF;IAClB,EAAE,OAAOG,KAAK;QACZ,IAAI;YACFN,WAAWK;QACb,EAAE,OAAM;QACN,8DAA8D;QAChE;QACA,MAAMC;IACR;AACF"}

package/dist/esm/store/duckdb/batch.js

@@ -4,8 +4,9 @@
 // not just the async crossing to reach them, so this recognizes the three statement shapes the
 // store issues (INSERT ... VALUES (...), UPDATE ... SET ... WHERE ..., DELETE ... WHERE ...) and
 // rewrites each into a multi-row form, verified against a live DuckDB connection at
-// implementation time (principle 8). Unrecognized SQL returns null; connection.ts's runBatch
-// falls back to a prepare-once, bind-and-run loop -- still one call from outside the store.
+// implementation time (PRINCIPLES: documented-means-tested). Unrecognized SQL returns null;
+// connection.ts's runBatch falls back to a prepare-once, bind-and-run loop -- still one call
+// from outside the store.
 //
 // This only ever runs against sense's own SQL literals (reconcile, the feature hooks), never
 // arbitrary user SQL (`sense sql` is a separate, unrewritten passthrough), so the shapes below

package/dist/esm/store/duckdb/batch.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites a single-row parameterized statement -- the shape every write loop in this codebase\n// prepares once and calls with N different param sets -- into one statement DuckDB executes over\n// all N rows in a single round trip. A columnar engine's cost is row-at-a-time writes themselves,\n// not just the async crossing to reach them, so this recognizes the three statement shapes the\n// store issues (INSERT ... VALUES (...), UPDATE ... SET ... WHERE ..., DELETE ... WHERE ...) and\n// rewrites each into a multi-row form, verified against a live DuckDB connection at\n// implementation time (principle 8). Unrecognized SQL returns null; connection.ts's runBatch\n// falls back to a prepare-once, bind-and-run loop -- still one call from outside the store.\n//\n// This only ever runs against sense's own SQL literals (reconcile, the feature hooks), never\n// arbitrary user SQL (`sense sql` is a separate, unrewritten passthrough), so the shapes below\n// are exactly what those call sites emit, not a general parser.\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. Only ever called on clauses already\n// shaped that way by the regexes above; a clause that doesn't fit yields a column list whose\n// length won't match the caller's row width, which rewriteUpdate/rewriteDelete then reject.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["INSERT_RE","UPDATE_RE","DELETE_RE","placeholderCount","tuple","match","length","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","join","rewriteInsert","sql","head","tail","rows","rewriteUpdate","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,kGAAkG;AAClG,+FAA+F;AAC/F,iGAAiG;AACjG,oFAAoF;AACpF,6FAA6F;AAC7F,4FAA4F;AAC5F,EAAE;AACF,6FAA6F;AAC7F,+FAA+F;AAC/F,gEAAgE;AAEhE,MAAMA,YAAY;AAClB,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,4FAA4F;AAC5F,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEnB,QAAQgB;IAAM,GAAG,IAAM,KAAKI,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOF,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAME,KAAKG,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIjB,UAAUkB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGY,MAAMzB,OAAO0B,KAAK,GAAGb;IAC9B,MAAMK,QAAQnB,iBAAiBC;IAC/B,IAAIkB,UAAU,GAAG,OAAO;IACxB,MAAMS,OAAOP,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEjB,MAAM,CAAC,CAAC,EAAEsB,IAAI,CAAC;IACvE,OAAO;QAAEE,KAAK,GAAGC,OAAOE,OAAOD,MAAM;QAAER;IAAM;AAC/C;AAEA,OAAO,SAASU,cAAcJ,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIhB,UAAUiB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOC,WAAWC,YAAY,GAAGlB;IAC1C,MAAMmB,UAAU7B,gBAAgB2B,WAAW;IAC3C,MAAMG,YAAY9B,gBAAgB4B,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQxB,GAAG,CAAC,CAAC4B,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IAC9D,MAAMe,WAAWJ,UAAUzB,GAAG,CAAC,CAAC4B,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEK,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEnB,OAAOC,UAAUiB,SAAShC,MAAM,EAAE,UAAU,EAAEgC,SAASZ,IAAI,CAAC,MAAM,QAAQ,EAAEe,UAAU;QAAEnB,OAAOgB,SAAShC,MAAM;IAAC;AAChL;AAEA,OAAO,SAASoC,cAAcd,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIf,UAAUgB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOE,YAAY,GAAGlB;IAC/B,MAAMF,OAAOR,gBAAgB4B,aAAa;IAC1C,IAAI,CAACpB,MAAM,OAAO;IAClB,IAAIA,KAAKT,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEsB,KAAK,CAAC,YAAY,EAAEK,MAAM,OAAO,EAAElB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEnB,QAAQe;YAAS,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEJ,OAAO;QAAE;IACjI;IACA,OAAO;QAAEM,KAAK,CAAC,YAAY,EAAEK,MAAM,QAAQ,EAAElB,KAAKW,IAAI,CAAC,MAAM,aAAa,EAAEN,OAAOC,UAAUN,KAAKT,MAAM,EAAE,CAAC,CAAC;QAAEgB,OAAOP,KAAKT,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASqC,aAAaf,GAAW,EAAEP,QAAgB;QACjDM,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcC,KAAKP,uBAAnBM,4BAAAA,iBAAgCK,cAAcJ,KAAKP,uBAAnDM,kBAAAA,OAAgEe,cAAcd,KAAKP;AAC5F"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/batch.ts"],"sourcesContent":["// Rewrites a single-row parameterized statement -- the shape every write loop in this codebase\n// prepares once and calls with N different param sets -- into one statement DuckDB executes over\n// all N rows in a single round trip. A columnar engine's cost is row-at-a-time writes themselves,\n// not just the async crossing to reach them, so this recognizes the three statement shapes the\n// store issues (INSERT ... VALUES (...), UPDATE ... SET ... WHERE ..., DELETE ... WHERE ...) and\n// rewrites each into a multi-row form, verified against a live DuckDB connection at\n// implementation time (PRINCIPLES: documented-means-tested). Unrecognized SQL returns null;\n// connection.ts's runBatch falls back to a prepare-once, bind-and-run loop -- still one call\n// from outside the store.\n//\n// This only ever runs against sense's own SQL literals (reconcile, the feature hooks), never\n// arbitrary user SQL (`sense sql` is a separate, unrewritten passthrough), so the shapes below\n// are exactly what those call sites emit, not a general parser.\n\nconst INSERT_RE = /^(INSERT\\s+(?:OR\\s+\\w+\\s+)?INTO\\s+\\S+\\s*\\([^)]*\\)\\s*VALUES\\s*)\\(([^()]*)\\)(.*)$/is;\nconst UPDATE_RE = /^UPDATE\\s+(\\S+)\\s+SET\\s+(.+?)\\s+WHERE\\s+(.+)$/is;\nconst DELETE_RE = /^DELETE\\s+FROM\\s+(\\S+)\\s+WHERE\\s+(.+)$/is;\n\nfunction placeholderCount(tuple: string): number {\n  return (tuple.match(/\\?/g) ?? []).length;\n}\n\n// \"a = ?, b = ?\" or \"a = ? AND b = ?\" -> [\"a\", \"b\"]. Only ever called on clauses already\n// shaped that way by the regexes above; a clause that doesn't fit yields a column list whose\n// length won't match the caller's row width, which rewriteUpdate/rewriteDelete then reject.\nfunction equalityColumns(clause: string, sep: RegExp): string[] | null {\n  const parts = clause.split(sep).map((p) => p.trim());\n  const cols: string[] = [];\n  for (const part of parts) {\n    const m = /^(.+?)=\\s*\\?$/.exec(part);\n    if (!m) return null;\n    cols.push(m[1].trim());\n  }\n  return cols;\n}\n\nfunction tuples(rowCount: number, width: number): string {\n  const row = `(${Array.from({ length: width }, () => '?').join(', ')})`;\n  return Array.from({ length: rowCount }, () => row).join(', ');\n}\n\nexport function rewriteInsert(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = INSERT_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, head, tuple, tail] = m;\n  const width = placeholderCount(tuple);\n  if (width === 0) return null;\n  const rows = Array.from({ length: rowCount }, () => `(${tuple})`).join(', ');\n  return { sql: `${head}${rows}${tail}`, width };\n}\n\nexport function rewriteUpdate(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = UPDATE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, setClause, whereClause] = m;\n  const setCols = equalityColumns(setClause, /,/);\n  const whereCols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!setCols || !whereCols) return null;\n  const dataCols = [...setCols, ...whereCols];\n  const setAssign = setCols.map((c) => `${c} = data.${c}`).join(', ');\n  const joinCond = whereCols.map((c) => `${table}.${c} = data.${c}`).join(' AND ');\n  return { sql: `UPDATE ${table} SET ${setAssign} FROM (VALUES ${tuples(rowCount, dataCols.length)}) AS data(${dataCols.join(', ')}) WHERE ${joinCond}`, width: dataCols.length };\n}\n\nexport function rewriteDelete(sql: string, rowCount: number): { sql: string; width: number } | null {\n  const m = DELETE_RE.exec(sql.trim());\n  if (!m) return null;\n  const [, table, whereClause] = m;\n  const cols = equalityColumns(whereClause, /\\bAND\\b/i);\n  if (!cols) return null;\n  if (cols.length === 1) {\n    return { sql: `DELETE FROM ${table} WHERE ${cols[0]} IN (${Array.from({ length: rowCount }, () => '?').join(', ')})`, width: 1 };\n  }\n  return { sql: `DELETE FROM ${table} WHERE (${cols.join(', ')}) IN (VALUES ${tuples(rowCount, cols.length)})`, width: cols.length };\n}\n\n// Tries each shape in turn; null means the caller should fall back to a per-row loop.\nexport function rewriteBatch(sql: string, rowCount: number): { sql: string; width: number } | null {\n  return rewriteInsert(sql, rowCount) ?? rewriteUpdate(sql, rowCount) ?? rewriteDelete(sql, rowCount);\n}\n"],"names":["INSERT_RE","UPDATE_RE","DELETE_RE","placeholderCount","tuple","match","length","equalityColumns","clause","sep","parts","split","map","p","trim","cols","part","m","exec","push","tuples","rowCount","width","row","Array","from","join","rewriteInsert","sql","head","tail","rows","rewriteUpdate","table","setClause","whereClause","setCols","whereCols","dataCols","setAssign","c","joinCond","rewriteDelete","rewriteBatch"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,kGAAkG;AAClG,+FAA+F;AAC/F,iGAAiG;AACjG,oFAAoF;AACpF,4FAA4F;AAC5F,6FAA6F;AAC7F,0BAA0B;AAC1B,EAAE;AACF,6FAA6F;AAC7F,+FAA+F;AAC/F,gEAAgE;AAEhE,MAAMA,YAAY;AAClB,MAAMC,YAAY;AAClB,MAAMC,YAAY;AAElB,SAASC,iBAAiBC,KAAa;QAC7BA;IAAR,OAAO,EAACA,eAAAA,MAAMC,KAAK,CAAC,oBAAZD,0BAAAA,eAAsB,EAAE,EAAEE,MAAM;AAC1C;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,4FAA4F;AAC5F,SAASC,gBAAgBC,MAAc,EAAEC,GAAW;IAClD,MAAMC,QAAQF,OAAOG,KAAK,CAACF,KAAKG,GAAG,CAAC,CAACC,IAAMA,EAAEC,IAAI;IACjD,MAAMC,OAAiB,EAAE;IACzB,KAAK,MAAMC,QAAQN,MAAO;QACxB,MAAMO,IAAI,gBAAgBC,IAAI,CAACF;QAC/B,IAAI,CAACC,GAAG,OAAO;QACfF,KAAKI,IAAI,CAACF,CAAC,CAAC,EAAE,CAACH,IAAI;IACrB;IACA,OAAOC;AACT;AAEA,SAASK,OAAOC,QAAgB,EAAEC,KAAa;IAC7C,MAAMC,MAAM,CAAC,CAAC,EAAEC,MAAMC,IAAI,CAAC;QAAEnB,QAAQgB;IAAM,GAAG,IAAM,KAAKI,IAAI,CAAC,MAAM,CAAC,CAAC;IACtE,OAAOF,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAME,KAAKG,IAAI,CAAC;AAC1D;AAEA,OAAO,SAASC,cAAcC,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIjB,UAAUkB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGY,MAAMzB,OAAO0B,KAAK,GAAGb;IAC9B,MAAMK,QAAQnB,iBAAiBC;IAC/B,IAAIkB,UAAU,GAAG,OAAO;IACxB,MAAMS,OAAOP,MAAMC,IAAI,CAAC;QAAEnB,QAAQe;IAAS,GAAG,IAAM,CAAC,CAAC,EAAEjB,MAAM,CAAC,CAAC,EAAEsB,IAAI,CAAC;IACvE,OAAO;QAAEE,KAAK,GAAGC,OAAOE,OAAOD,MAAM;QAAER;IAAM;AAC/C;AAEA,OAAO,SAASU,cAAcJ,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIhB,UAAUiB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOC,WAAWC,YAAY,GAAGlB;IAC1C,MAAMmB,UAAU7B,gBAAgB2B,WAAW;IAC3C,MAAMG,YAAY9B,gBAAgB4B,aAAa;IAC/C,IAAI,CAACC,WAAW,CAACC,WAAW,OAAO;IACnC,MAAMC,WAAW;WAAIF;WAAYC;KAAU;IAC3C,MAAME,YAAYH,QAAQxB,GAAG,CAAC,CAAC4B,IAAM,GAAGA,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IAC9D,MAAMe,WAAWJ,UAAUzB,GAAG,CAAC,CAAC4B,IAAM,GAAGP,MAAM,CAAC,EAAEO,EAAE,QAAQ,EAAEA,GAAG,EAAEd,IAAI,CAAC;IACxE,OAAO;QAAEE,KAAK,CAAC,OAAO,EAAEK,MAAM,KAAK,EAAEM,UAAU,cAAc,EAAEnB,OAAOC,UAAUiB,SAAShC,MAAM,EAAE,UAAU,EAAEgC,SAASZ,IAAI,CAAC,MAAM,QAAQ,EAAEe,UAAU;QAAEnB,OAAOgB,SAAShC,MAAM;IAAC;AAChL;AAEA,OAAO,SAASoC,cAAcd,GAAW,EAAEP,QAAgB;IACzD,MAAMJ,IAAIf,UAAUgB,IAAI,CAACU,IAAId,IAAI;IACjC,IAAI,CAACG,GAAG,OAAO;IACf,MAAM,GAAGgB,OAAOE,YAAY,GAAGlB;IAC/B,MAAMF,OAAOR,gBAAgB4B,aAAa;IAC1C,IAAI,CAACpB,MAAM,OAAO;IAClB,IAAIA,KAAKT,MAAM,KAAK,GAAG;QACrB,OAAO;YAAEsB,KAAK,CAAC,YAAY,EAAEK,MAAM,OAAO,EAAElB,IAAI,CAAC,EAAE,CAAC,KAAK,EAAES,MAAMC,IAAI,CAAC;gBAAEnB,QAAQe;YAAS,GAAG,IAAM,KAAKK,IAAI,CAAC,MAAM,CAAC,CAAC;YAAEJ,OAAO;QAAE;IACjI;IACA,OAAO;QAAEM,KAAK,CAAC,YAAY,EAAEK,MAAM,QAAQ,EAAElB,KAAKW,IAAI,CAAC,MAAM,aAAa,EAAEN,OAAOC,UAAUN,KAAKT,MAAM,EAAE,CAAC,CAAC;QAAEgB,OAAOP,KAAKT,MAAM;IAAC;AACnI;AAEA,sFAAsF;AACtF,OAAO,SAASqC,aAAaf,GAAW,EAAEP,QAAgB;QACjDM,MAAAA;IAAP,QAAOA,QAAAA,iBAAAA,cAAcC,KAAKP,uBAAnBM,4BAAAA,iBAAgCK,cAAcJ,KAAKP,uBAAnDM,kBAAAA,OAAgEe,cAAcd,KAAKP;AAC5F"}

package/dist/esm/store/duckdb/connection.js

@@ -12,6 +12,17 @@ export function storeRowToJs(row) {
     for (const [key, value] of Object.entries(row))out[key] = storeValueToJs(value);
     return out;
 }
+// @duckdb/node-api tracks every prepared statement itself and destroys any survivor when the
+// connection closes, but sense's stores stay open for a whole `sense watch` session -- nothing
+// destroys one before then. The portable Statement interface has no dispose (sqlite's node:sqlite
+// statements need none, and callers like graph/traverse.ts's ring loop reuse one Statement across
+// several run/get/all calls, so a per-call destroy would be wrong), so cleanup happens here,
+// locally, once nothing references the wrapper any more. destroySync() is safe to call more than
+// once -- runBatch below and vectors.ts already call it explicitly while the same connection-close
+// sweep can also reach a statement that outlives its wrapper, with no reported issue.
+const preparedFinalizer = new FinalizationRegistry((prepared)=>{
+    prepared.destroySync();
+});
 // Wraps one already-prepared DuckDB statement: prepare() is the only async step (Store.prepare
 // already crosses the async boundary once), so columns() reads back synchronously from it and
 // run/get/all rebind and re-execute the same native statement, matching how a caller (e.g.
@@ -52,6 +63,7 @@ let DuckdbStatement = class DuckdbStatement {
     }
     constructor(prepared){
         this.prepared = prepared;
+        preparedFinalizer.register(this, prepared);
     }
 };
 export function createConnection(duckdb) {

package/dist/esm/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's\n// small ints are numbers and consumers assume number. Beyond the safe range stays BigInt (the\n// setReadBigInts contract); stringifyJson keeps JSON output safe.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step (Store.prepare\n// already crosses the async boundary once), so columns() reads back synchronously from it and\n// run/get/all rebind and re-execute the same native statement, matching how a caller (e.g.\n// graph/traverse.ts's ring loop) reuses one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch turns the caller's single-row SQL into\n    // one multi-row statement when it recognizes the shape (see batch.ts); anything else falls\n    // back to a bind-and-run loop over one prepared statement, still a single call to the caller.\n    // Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so\n    // a standalone batch stays atomic.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","DuckdbStatement","run","params","length","prepared","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat","destroySync"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,0FAA0F;AAC1F,8FAA8F;AAC9F,kEAAkE;AAClE,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,+FAA+F;AAC/F,8FAA8F;AAC9F,2FAA2F;AAC3F,4EAA4E;AAC5E,IAAA,AAAMI,kBAAN,MAAMA;IAOJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACC,QAAQ,CAACC,IAAI,CAACH;QAC1C,MAAMI,SAAS,MAAM,IAAI,CAACF,QAAQ,CAACH,GAAG;QACtC,OAAO;YAAEM,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGR,MAAiB,EAAoB;QAChD,MAAMS,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIV;QAC/B,OAAOS,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGV,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACC,QAAQ,CAACC,IAAI,CAACH;QAC1C,MAAMW,SAAS,MAAM,IAAI,CAACT,QAAQ,CAACU,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACtB;IAC1E;IAEA,OAAOuB,QAAQ,GAAGf,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACU,GAAG,IAAIV;IAC3B;IAEAgB,UAAmC;QACjC,MAAMtB,MAA+B,EAAE;QACvC,IAAK,IAAIuB,IAAI,GAAGA,IAAI,IAAI,CAACf,QAAQ,CAACgB,WAAW,EAAED,IAAKvB,IAAIyB,IAAI,CAAC;YAAEC,MAAM,IAAI,CAAClB,QAAQ,CAACmB,UAAU,CAACJ;QAAG;QACjG,OAAOvB;IACT;IAEA4B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IApCA,YAAYrB,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;IAClB;AAmCF;AAEA,OAAO,SAASsB,iBAAiBC,MAAwB;IACvD,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,4FAA4F;QAC5F,2FAA2F;QAC3F,8FAA8F;QAC9F,4FAA4F;QAC5F,mCAAmC;QACnC,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMjB,gBAAgB0C,MAAM;gBAC1B,MAAMM,YAAY/C,aAAa2C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK9B,IAAI,CAAC4B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKE,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMF,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMnC,OAAOsC,UAAW;wBAC3BE,KAAK9B,IAAI,CAACV;wBACV,MAAMwC,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKE,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAOT;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's\n// small ints are numbers and consumers assume number. Beyond the safe range stays BigInt (the\n// setReadBigInts contract); stringifyJson keeps JSON output safe.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api tracks every prepared statement itself and destroys any survivor when the\n// connection closes, but sense's stores stay open for a whole `sense watch` session -- nothing\n// destroys one before then. The portable Statement interface has no dispose (sqlite's node:sqlite\n// statements need none, and callers like graph/traverse.ts's ring loop reuse one Statement across\n// several run/get/all calls, so a per-call destroy would be wrong), so cleanup happens here,\n// locally, once nothing references the wrapper any more. destroySync() is safe to call more than\n// once -- runBatch below and vectors.ts already call it explicitly while the same connection-close\n// sweep can also reach a statement that outlives its wrapper, with no reported issue.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step (Store.prepare\n// already crosses the async boundary once), so columns() reads back synchronously from it and\n// run/get/all rebind and re-execute the same native statement, matching how a caller (e.g.\n// graph/traverse.ts's ring loop) reuses one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch turns the caller's single-row SQL into\n    // one multi-row statement when it recognizes the shape (see batch.ts); anything else falls\n    // back to a bind-and-run loop over one prepared statement, still a single call to the caller.\n    // Joins the caller's transaction when there is one (reconcile); otherwise opens its own, so\n    // a standalone batch stays atomic.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,0FAA0F;AAC1F,8FAA8F;AAC9F,kEAAkE;AAClE,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,kGAAkG;AAClG,kGAAkG;AAClG,6FAA6F;AAC7F,iGAAiG;AACjG,mGAAmG;AACnG,sFAAsF;AACtF,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,+FAA+F;AAC/F,8FAA8F;AAC9F,2FAA2F;AAC3F,4EAA4E;AAC5E,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAEA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,4FAA4F;QAC5F,2FAA2F;QAC3F,8FAA8F;QAC9F,4FAA4F;QAC5F,mCAAmC;QACnC,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}

package/dist/esm/store/duckdb/fieldStats.d.ts

@@ -0,0 +1,2 @@
+import type { Connection, FieldStat } from '../types.js';
+export declare function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]>;

package/dist/esm/store/duckdb/fieldStats.js

@@ -0,0 +1,39 @@
+import { quoteIdent } from '../shared.js';
+// DuckDB's frontmatter columns are VARIANT (reconcile.ts's comment on why), so typeof() reports
+// VARIANT for every field regardless of the boxed value; variant_typeof() reports the boxed
+// native type instead (spike-verified against mapValue()'s actual bind shapes: a bound JS bigint
+// always boxes as INT128, a non-integer JS number always boxes as DOUBLE, a string as VARCHAR --
+// literal SQL values box narrower, e.g. INTEGER/DECIMAL(p, s), which this still covers since a
+// query issued through `sense sql` can produce either). Mapped onto the shared integer/real/text
+// vocabulary sqlite/turso's typeof() already emits natively; an unrecognized name throws rather
+// than being silently called text (no-silent-modes).
+function classifyVariantType(name, field) {
+    if (/^U?INT\d+$/.test(name)) return 'integer';
+    if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';
+    if (name === 'VARCHAR') return 'text';
+    throw new Error(`duckdb: variant_typeof() returned unrecognized type "${name}" for column "${field}"`);
+}
+// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per
+// column, so the query stays O(columns) rather than O(rows x columns). '|' is the string_agg
+// separator, not ',': DECIMAL's own display name ("DECIMAL(4, 2)") contains a comma, which a
+// ',' separator could not be split back apart from.
+export async function fieldStats(conn, columns, scopeWhere) {
+    const exprs = columns.map((name, i)=>{
+        const quoted = quoteIdent(name);
+        return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;
+    });
+    const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);
+    const row = await stmt.get();
+    return columns.map((name, i)=>{
+        var _row_, _row_1;
+        const raw = ((_row_ = row[`t${i}`]) !== null && _row_ !== void 0 ? _row_ : '').split('|').filter(Boolean);
+        const types = [
+            ...new Set(raw.map((t)=>classifyVariantType(t, name)))
+        ].sort();
+        return {
+            field: name,
+            coverage: Number((_row_1 = row[`n${i}`]) !== null && _row_1 !== void 0 ? _row_1 : 0),
+            type: types.join(',')
+        };
+    });
+}

package/dist/esm/store/duckdb/fieldStats.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/fieldStats.ts"],"sourcesContent":["import { quoteIdent } from '../shared.ts';\nimport type { Connection, FieldStat } from '../types.ts';\n\n// DuckDB's frontmatter columns are VARIANT (reconcile.ts's comment on why), so typeof() reports\n// VARIANT for every field regardless of the boxed value; variant_typeof() reports the boxed\n// native type instead (spike-verified against mapValue()'s actual bind shapes: a bound JS bigint\n// always boxes as INT128, a non-integer JS number always boxes as DOUBLE, a string as VARCHAR --\n// literal SQL values box narrower, e.g. INTEGER/DECIMAL(p, s), which this still covers since a\n// query issued through `sense sql` can produce either). Mapped onto the shared integer/real/text\n// vocabulary sqlite/turso's typeof() already emits natively; an unrecognized name throws rather\n// than being silently called text (no-silent-modes).\nfunction classifyVariantType(name: string, field: string): 'integer' | 'real' | 'text' {\n  if (/^U?INT\\d+$/.test(name)) return 'integer';\n  if (name === 'FLOAT' || name === 'DOUBLE' || name.startsWith('DECIMAL(')) return 'real';\n  if (name === 'VARCHAR') return 'text';\n  throw new Error(`duckdb: variant_typeof() returned unrecognized type \"${name}\" for column \"${field}\"`);\n}\n\n// One aggregate query, one result row: COUNT + string_agg(DISTINCT variant_typeof()) per\n// column, so the query stays O(columns) rather than O(rows x columns). '|' is the string_agg\n// separator, not ',': DECIMAL's own display name (\"DECIMAL(4, 2)\") contains a comma, which a\n// ',' separator could not be split back apart from.\nexport async function fieldStats(conn: Connection, columns: string[], scopeWhere: string): Promise<FieldStat[]> {\n  const exprs = columns.map((name, i) => {\n    const quoted = quoteIdent(name);\n    return `COUNT(${quoted}) AS n${i}, string_agg(DISTINCT variant_typeof(${quoted}), '|') FILTER (WHERE ${quoted} IS NOT NULL) AS t${i}`;\n  });\n  const stmt = await conn.prepare(`SELECT ${exprs.join(', ')} FROM frontmatter ${scopeWhere}`);\n  const row = (await stmt.get()) as Record<string, number | string | null>;\n  return columns.map((name, i) => {\n    const raw = ((row[`t${i}`] as string) ?? '').split('|').filter(Boolean);\n    const types = [...new Set(raw.map((t) => classifyVariantType(t, name)))].sort();\n    return { field: name, coverage: Number(row[`n${i}`] ?? 0), type: types.join(',') };\n  });\n}\n"],"names":["quoteIdent","classifyVariantType","name","field","test","startsWith","Error","fieldStats","conn","columns","scopeWhere","exprs","map","i","quoted","stmt","prepare","join","row","get","raw","split","filter","Boolean","types","Set","t","sort","coverage","Number","type"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,gGAAgG;AAChG,4FAA4F;AAC5F,iGAAiG;AACjG,iGAAiG;AACjG,+FAA+F;AAC/F,iGAAiG;AACjG,gGAAgG;AAChG,qDAAqD;AACrD,SAASC,oBAAoBC,IAAY,EAAEC,KAAa;IACtD,IAAI,aAAaC,IAAI,CAACF,OAAO,OAAO;IACpC,IAAIA,SAAS,WAAWA,SAAS,YAAYA,KAAKG,UAAU,CAAC,aAAa,OAAO;IACjF,IAAIH,SAAS,WAAW,OAAO;IAC/B,MAAM,IAAII,MAAM,CAAC,qDAAqD,EAAEJ,KAAK,cAAc,EAAEC,MAAM,CAAC,CAAC;AACvG;AAEA,yFAAyF;AACzF,6FAA6F;AAC7F,6FAA6F;AAC7F,oDAAoD;AACpD,OAAO,eAAeI,WAAWC,IAAgB,EAAEC,OAAiB,EAAEC,UAAkB;IACtF,MAAMC,QAAQF,QAAQG,GAAG,CAAC,CAACV,MAAMW;QAC/B,MAAMC,SAASd,WAAWE;QAC1B,OAAO,CAAC,MAAM,EAAEY,OAAO,MAAM,EAAED,EAAE,qCAAqC,EAAEC,OAAO,sBAAsB,EAAEA,OAAO,kBAAkB,EAAED,GAAG;IACvI;IACA,MAAME,OAAO,MAAMP,KAAKQ,OAAO,CAAC,CAAC,OAAO,EAAEL,MAAMM,IAAI,CAAC,MAAM,kBAAkB,EAAEP,YAAY;IAC3F,MAAMQ,MAAO,MAAMH,KAAKI,GAAG;IAC3B,OAAOV,QAAQG,GAAG,CAAC,CAACV,MAAMW;YACVK,OAEyBA;QAFvC,MAAME,MAAM,EAAEF,QAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,mBAAAA,QAA2B,IAAIG,KAAK,CAAC,KAAKC,MAAM,CAACC;QAC/D,MAAMC,QAAQ;eAAI,IAAIC,IAAIL,IAAIR,GAAG,CAAC,CAACc,IAAMzB,oBAAoByB,GAAGxB;SAAQ,CAACyB,IAAI;QAC7E,OAAO;YAAExB,OAAOD;YAAM0B,UAAUC,QAAOX,SAAAA,GAAG,CAAC,CAAC,CAAC,EAAEL,GAAG,CAAC,cAAZK,oBAAAA,SAAgB;YAAIY,MAAMN,MAAMP,IAAI,CAAC;QAAK;IACnF;AACF"}

package/dist/esm/store/duckdb/lexical.js

@@ -18,8 +18,9 @@ const FIELD_WEIGHT = {
 };
 const UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');
 // FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would
-// otherwise silently treat as literal terms instead of honoring (PRINCIPLES.md #6). Checked
-// against terms with quoted spans blanked -- those go through contains() and are supported.
+// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).
+// Checked against terms with quoted spans blanked -- those go through contains() and are
+// supported.
 const FTS5_OPERATORS = [
     {
         label: 'prefix query',
@@ -95,33 +96,42 @@ async function ensureFtsFresh(conn, state) {
 }
 // Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both
 // (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are
-// emitted -- positional `?` binding requires that order to match the assembled SQL text.
+// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score
+// and gate params are collected apart and joined at the end because the assembled SQL puts
+// every score part ahead of every gate part: appending to one array term by term matches that
+// text only while the query has words or substrings, never both, and silently shifts each `?`
+// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a
+// different question rather than failing).
 function buildScoreAndGate(words, substrings) {
     const scoreParts = [];
     const gateParts = [];
-    const params = [];
+    const scoreParams = [];
+    const gateParams = [];
     if (words.length > 0) {
         const wordQuery = words.join(' ');
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content."path", ?, fields := '${field}', conjunctive := false), 0)`);
-            params.push(wordQuery);
+            scoreParams.push(wordQuery);
         }
         gateParts.push(`fts_main_content.match_bm25(content."path", ?, conjunctive := true) IS NOT NULL`);
-        params.push(wordQuery);
+        gateParams.push(wordQuery);
     }
     for (const raw of substrings){
         const needle = raw.toLowerCase();
         for (const field of FIELDS){
             scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);
-            params.push(needle);
+            scoreParams.push(needle);
         }
         gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);
-        params.push(needle);
+        gateParams.push(needle);
     }
     return {
         scoreSql: scoreParts.join(' + '),
         gateSql: gateParts.join(' AND '),
-        params
+        params: [
+            ...scoreParams,
+            ...gateParams
+        ]
     };
 }
 // Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's

package/dist/esm/store/duckdb/lexical.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES.md #6). Checked\n// against terms with quoted spans blanked -- those go through contains() and are supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text.\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const params: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      params.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    params.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      params.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    params.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","params","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,4FAA4F;AAC5F,4FAA4F;AAC5F,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,yFAAyF;AACzF,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,SAAoB,EAAE;IAE5B,IAAInB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMe,YAAYpB,MAAMqB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJH,OAAOb,IAAI,CAACc;QACd;QACAF,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGa,OAAOb,IAAI,CAACc;IACd;IAEA,KAAK,MAAMG,OAAOtB,WAAY;QAC5B,MAAMuB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAASzC,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACwC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGH,OAAOb,IAAI,CAACkB;QACd;QACAN,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHa,OAAOb,IAAI,CAACkB;IACd;IAEA,OAAO;QAAEE,UAAUT,WAAWI,IAAI,CAAC;QAAQM,SAAST,UAAUG,IAAI,CAAC;QAAUF;IAAO;AACtF;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeS,aAAahB,IAAgB,EAAEpB,KAAa,EAAEqC,IAAyB,EAAEhB,KAAoB;IACjH,MAAMiB,cAAcvC,oBAAoBC;IACxC,IAAIsC,gBAAgB,MAAM;QACxB,MAAM,IAAIpD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEoD,YAAYzC,KAAK,CAAC,GAAG,EAAEyC,YAAYjC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEiC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE7B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEa,QAAQ,EAAEC,OAAO,EAAER,MAAM,EAAE,GAAGH,kBAAkBhB,OAAOC;IAC/D,MAAMkC,MAAM,CAAC;oCACqB,EAAET,SAAS;;IAE3C,EAAEK,UAAU;UACN,EAAEJ,QAAQ;IAChB,EAAEK,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAMxB,KAAKyB,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAInB,QAAQe;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB3B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL0B,OAAO,CAAChD,OAAOqC,OAASD,aAAahB,MAAMpB,OAAOqC,MAAMhB;QACxD4B,WAAW;YACT5B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/lexical.ts"],"sourcesContent":["// D1: fts BM25 for ranking, contains() scans for exact substring / phrase verification /\n// unspaced scripts, JS excerpts for every doc (no snippet()).\nimport { SenseError } from '../../errors.ts';\nimport { UNSPACED_SCRIPTS } from '../../text/segment.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, LexicalHit, LexicalQueryOptions } from '../types.ts';\n\nconst FIELDS = ['title', 'summary', 'text'] as const;\ntype Field = (typeof FIELDS)[number];\n// Mirrors sqlite's bm25(content, 10.0, 5.0, 1.0, ...) column-weight intent; DuckDB's\n// match_bm25 has no field_weights argument (verified 1.5.5), so each field is scored\n// separately and combined here.\nconst FIELD_WEIGHT: Record<Field, number> = { title: 10, summary: 5, text: 1 };\n\nconst UNSPACED_RUN = new RegExp(`[${UNSPACED_SCRIPTS}]`, 'u');\n\n// FTS5 operator syntax (sqlite.org/fts5.html sec. 3) that words/substrings below would\n// otherwise silently treat as literal terms instead of honoring (PRINCIPLES: no-silent-modes).\n// Checked against terms with quoted spans blanked -- those go through contains() and are\n// supported.\nconst FTS5_OPERATORS: Array<{ label: string; re: RegExp }> = [\n  { label: 'prefix query', re: /[\\p{L}\\p{N}_]+\\*(?=\\s|$)/u },\n  { label: 'boolean operator', re: /(?:^|\\s)(?:AND|OR|NOT)(?=\\s|$)/ },\n  { label: 'NEAR operator', re: /(?:^|\\s)NEAR\\b/ },\n  { label: 'initial-token operator', re: /(?:^|\\s)\\^\\S+/ },\n  { label: 'column filter', re: /(?:^|\\s)[\\p{L}_]\\w*\\s*:/u },\n];\n\nfunction unsupportedOperator(terms: string): { label: string; token: string } | null {\n  const withoutPhrases = terms.replace(/\"[^\"]*\"/g, ' ');\n  for (const { label, re } of FTS5_OPERATORS) {\n    const m = withoutPhrases.match(re);\n    if (m) return { label, token: m[0].trim() };\n  }\n  return null;\n}\n\ninterface FtsIndexState {\n  stale: boolean;\n}\n\n// A run whose script marks no word boundaries makes match_bm25's whitespace tokenizer index\n// the whole run as one token (same gap FTS5 has without the `_seg` sidecar), so such runs --\n// and any author-quoted phrase, any script -- go through contains() instead of the fts index.\nfunction splitTerms(terms: string): { words: string[]; substrings: string[] } {\n  const words: string[] = [];\n  const substrings: string[] = [];\n  const withoutPhrases = terms.replace(/\"([^\"]*)\"/g, (_m, inner: string) => {\n    const phrase = inner.trim();\n    if (phrase.length === 0) return ' ';\n    substrings.push(phrase);\n    if (!UNSPACED_RUN.test(phrase)) for (const w of phrase.split(/\\s+/)) if (w.length > 0) words.push(w);\n    return ' ';\n  });\n  for (const tok of withoutPhrases.split(/\\s+/)) {\n    if (tok.length === 0) continue;\n    if (UNSPACED_RUN.test(tok)) substrings.push(tok);\n    else words.push(tok);\n  }\n  return { words, substrings };\n}\n\n// No incremental update (verified 1.5.5: PRAGMA create_fts_index is rebuild-only), so this\n// pays the full rebuild -- but only once per store instance's first lexical query, and only\n// when content actually changed since (see FtsIndexState / duckdb/store.ts's markStale()).\nasync function ensureFtsFresh(conn: Connection, state: FtsIndexState): Promise<void> {\n  if (!state.stale) return;\n  await conn.exec('INSTALL fts; LOAD fts;');\n  await withTransaction(conn, async () => {\n    // stopwords='none': sqlite's porter/unicode61 tokenizer never removes stopwords either\n    // (verified 1.5.5), and the fts extension's default 571-word English list would otherwise\n    // silently drop common query words (e.g. \"and\") from the index but not from match_bm25's\n    // conjunctive gate, making a bare multi-word query that contains one match nothing.\n    await conn.exec(`PRAGMA create_fts_index('content', 'path', 'title', 'summary', 'text', stopwords='none', overwrite=1)`);\n  });\n  state.stale = false;\n}\n\n// Per-word-term and per-substring-term SQL fragments, field-weighted the same way for both\n// (title 10 / summary 5 / text 1), plus the params in the exact left-to-right order they are\n// emitted -- positional `?` binding requires that order to match the assembled SQL text. Score\n// and gate params are collected apart and joined at the end because the assembled SQL puts\n// every score part ahead of every gate part: appending to one array term by term matches that\n// text only while the query has words or substrings, never both, and silently shifts each `?`\n// by one as soon as it has both (PRINCIPLES: no-silent-modes -- a shifted bind answers a\n// different question rather than failing).\nfunction buildScoreAndGate(words: string[], substrings: string[]): { scoreSql: string; gateSql: string; params: unknown[] } {\n  const scoreParts: string[] = [];\n  const gateParts: string[] = [];\n  const scoreParams: unknown[] = [];\n  const gateParams: unknown[] = [];\n\n  if (words.length > 0) {\n    const wordQuery = words.join(' ');\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * COALESCE(fts_main_content.match_bm25(content.\"path\", ?, fields := '${field}', conjunctive := false), 0)`);\n      scoreParams.push(wordQuery);\n    }\n    gateParts.push(`fts_main_content.match_bm25(content.\"path\", ?, conjunctive := true) IS NOT NULL`);\n    gateParams.push(wordQuery);\n  }\n\n  for (const raw of substrings) {\n    const needle = raw.toLowerCase();\n    for (const field of FIELDS) {\n      scoreParts.push(`${FIELD_WEIGHT[field]}.0 * CAST(contains(lower(content.${field}), ?) AS INTEGER)`);\n      scoreParams.push(needle);\n    }\n    gateParts.push(`contains(lower(content.title) || ' ' || lower(content.summary) || ' ' || lower(content.text), ?)`);\n    gateParams.push(needle);\n  }\n\n  return { scoreSql: scoreParts.join(' + '), gateSql: gateParts.join(' AND '), params: [...scoreParams, ...gateParams] };\n}\n\n// Ranked word-match query, scoped by the caller-built SQL fragments (same fragments sqlite's\n// queryLexical takes). `hit` is always NULL: no snippet() equivalent exists, so every row goes\n// through the caller's JS excerpt fallback (commands/search.ts) rather than a second excerpt path.\nexport async function queryLexical(conn: Connection, terms: string, opts: LexicalQueryOptions, state: FtsIndexState): Promise<LexicalHit[]> {\n  const unsupported = unsupportedOperator(terms);\n  if (unsupported !== null) {\n    throw new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" does not implement FTS5's ${unsupported.label} (\"${unsupported.token}\") in this build; rephrase \"${terms.trim()}\" without it, or set \"store\" to \"sqlite\" in this tree's config to search it as written`);\n  }\n  const { whereJoin, whereCond, scopeCond, limit } = opts;\n  const { words, substrings } = splitTerms(terms);\n  if (words.length === 0 && substrings.length === 0) return [];\n\n  if (words.length > 0) await ensureFtsFresh(conn, state);\n\n  const { scoreSql, gateSql, params } = buildScoreAndGate(words, substrings);\n  const sql = `SELECT path, NULL AS hit FROM (\n    SELECT content.\"path\" AS path, (${scoreSql}) AS score\n    FROM content\n    ${whereJoin}\n    WHERE ${gateSql}\n    ${whereCond} ${scopeCond}\n  ) sq ORDER BY score DESC, path LIMIT ?`;\n  const stmt = await conn.prepare(sql);\n  return (await stmt.all(...params, limit)) as unknown as LexicalHit[];\n}\n\n// One instance per store: `stale` starts true (a fresh connection cannot know whether an\n// on-disk fts schema still matches `content`), and store.ts's reconcile wrapper flips it back\n// to true whenever content changes, so the next query rebuilds before ranking against it.\nexport function createLexicalIndex(conn: Connection): { query: (terms: string, opts: LexicalQueryOptions) => Promise<LexicalHit[]>; markStale: () => void } {\n  const state: FtsIndexState = { stale: true };\n  return {\n    query: (terms, opts) => queryLexical(conn, terms, opts, state),\n    markStale: () => {\n      state.stale = true;\n    },\n  };\n}\n"],"names":["SenseError","UNSPACED_SCRIPTS","withTransaction","FIELDS","FIELD_WEIGHT","title","summary","text","UNSPACED_RUN","RegExp","FTS5_OPERATORS","label","re","unsupportedOperator","terms","withoutPhrases","replace","m","match","token","trim","splitTerms","words","substrings","_m","inner","phrase","length","push","test","w","split","tok","ensureFtsFresh","conn","state","stale","exec","buildScoreAndGate","scoreParts","gateParts","scoreParams","gateParams","wordQuery","join","field","raw","needle","toLowerCase","scoreSql","gateSql","params","queryLexical","opts","unsupported","whereJoin","whereCond","scopeCond","limit","sql","stmt","prepare","all","createLexicalIndex","query","markStale"],"mappings":"AAAA,yFAAyF;AACzF,8DAA8D;AAC9D,SAASA,UAAU,QAAQ,kBAAkB;AAC7C,SAASC,gBAAgB,QAAQ,wBAAwB;AACzD,SAASC,eAAe,QAAQ,oBAAoB;AAGpD,MAAMC,SAAS;IAAC;IAAS;IAAW;CAAO;AAE3C,qFAAqF;AACrF,qFAAqF;AACrF,gCAAgC;AAChC,MAAMC,eAAsC;IAAEC,OAAO;IAAIC,SAAS;IAAGC,MAAM;AAAE;AAE7E,MAAMC,eAAe,IAAIC,OAAO,CAAC,CAAC,EAAER,iBAAiB,CAAC,CAAC,EAAE;AAEzD,uFAAuF;AACvF,+FAA+F;AAC/F,yFAAyF;AACzF,aAAa;AACb,MAAMS,iBAAuD;IAC3D;QAAEC,OAAO;QAAgBC,IAAI;IAA4B;IACzD;QAAED,OAAO;QAAoBC,IAAI;IAAiC;IAClE;QAAED,OAAO;QAAiBC,IAAI;IAAiB;IAC/C;QAAED,OAAO;QAA0BC,IAAI;IAAgB;IACvD;QAAED,OAAO;QAAiBC,IAAI;IAA2B;CAC1D;AAED,SAASC,oBAAoBC,KAAa;IACxC,MAAMC,iBAAiBD,MAAME,OAAO,CAAC,YAAY;IACjD,KAAK,MAAM,EAAEL,KAAK,EAAEC,EAAE,EAAE,IAAIF,eAAgB;QAC1C,MAAMO,IAAIF,eAAeG,KAAK,CAACN;QAC/B,IAAIK,GAAG,OAAO;YAAEN;YAAOQ,OAAOF,CAAC,CAAC,EAAE,CAACG,IAAI;QAAG;IAC5C;IACA,OAAO;AACT;AAMA,4FAA4F;AAC5F,6FAA6F;AAC7F,8FAA8F;AAC9F,SAASC,WAAWP,KAAa;IAC/B,MAAMQ,QAAkB,EAAE;IAC1B,MAAMC,aAAuB,EAAE;IAC/B,MAAMR,iBAAiBD,MAAME,OAAO,CAAC,cAAc,CAACQ,IAAIC;QACtD,MAAMC,SAASD,MAAML,IAAI;QACzB,IAAIM,OAAOC,MAAM,KAAK,GAAG,OAAO;QAChCJ,WAAWK,IAAI,CAACF;QAChB,IAAI,CAAClB,aAAaqB,IAAI,CAACH,SAAS;YAAA,KAAK,MAAMI,KAAKJ,OAAOK,KAAK,CAAC,OAAQ,IAAID,EAAEH,MAAM,GAAG,GAAGL,MAAMM,IAAI,CAACE;QAAE;QACpG,OAAO;IACT;IACA,KAAK,MAAME,OAAOjB,eAAegB,KAAK,CAAC,OAAQ;QAC7C,IAAIC,IAAIL,MAAM,KAAK,GAAG;QACtB,IAAInB,aAAaqB,IAAI,CAACG,MAAMT,WAAWK,IAAI,CAACI;aACvCV,MAAMM,IAAI,CAACI;IAClB;IACA,OAAO;QAAEV;QAAOC;IAAW;AAC7B;AAEA,2FAA2F;AAC3F,4FAA4F;AAC5F,2FAA2F;AAC3F,eAAeU,eAAeC,IAAgB,EAAEC,KAAoB;IAClE,IAAI,CAACA,MAAMC,KAAK,EAAE;IAClB,MAAMF,KAAKG,IAAI,CAAC;IAChB,MAAMnC,gBAAgBgC,MAAM;QAC1B,uFAAuF;QACvF,0FAA0F;QAC1F,yFAAyF;QACzF,oFAAoF;QACpF,MAAMA,KAAKG,IAAI,CAAC,CAAC,qGAAqG,CAAC;IACzH;IACAF,MAAMC,KAAK,GAAG;AAChB;AAEA,2FAA2F;AAC3F,6FAA6F;AAC7F,+FAA+F;AAC/F,2FAA2F;AAC3F,8FAA8F;AAC9F,8FAA8F;AAC9F,yFAAyF;AACzF,2CAA2C;AAC3C,SAASE,kBAAkBhB,KAAe,EAAEC,UAAoB;IAC9D,MAAMgB,aAAuB,EAAE;IAC/B,MAAMC,YAAsB,EAAE;IAC9B,MAAMC,cAAyB,EAAE;IACjC,MAAMC,aAAwB,EAAE;IAEhC,IAAIpB,MAAMK,MAAM,GAAG,GAAG;QACpB,MAAMgB,YAAYrB,MAAMsB,IAAI,CAAC;QAC7B,KAAK,MAAMC,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,wEAAwE,EAAEA,MAAM,4BAA4B,CAAC;YACpJJ,YAAYb,IAAI,CAACe;QACnB;QACAH,UAAUZ,IAAI,CAAC,CAAC,+EAA+E,CAAC;QAChGc,WAAWd,IAAI,CAACe;IAClB;IAEA,KAAK,MAAMG,OAAOvB,WAAY;QAC5B,MAAMwB,SAASD,IAAIE,WAAW;QAC9B,KAAK,MAAMH,SAAS1C,OAAQ;YAC1BoC,WAAWX,IAAI,CAAC,GAAGxB,YAAY,CAACyC,MAAM,CAAC,iCAAiC,EAAEA,MAAM,iBAAiB,CAAC;YAClGJ,YAAYb,IAAI,CAACmB;QACnB;QACAP,UAAUZ,IAAI,CAAC,CAAC,gGAAgG,CAAC;QACjHc,WAAWd,IAAI,CAACmB;IAClB;IAEA,OAAO;QAAEE,UAAUV,WAAWK,IAAI,CAAC;QAAQM,SAASV,UAAUI,IAAI,CAAC;QAAUO,QAAQ;eAAIV;eAAgBC;SAAW;IAAC;AACvH;AAEA,6FAA6F;AAC7F,+FAA+F;AAC/F,mGAAmG;AACnG,OAAO,eAAeU,aAAalB,IAAgB,EAAEpB,KAAa,EAAEuC,IAAyB,EAAElB,KAAoB;IACjH,MAAMmB,cAAczC,oBAAoBC;IACxC,IAAIwC,gBAAgB,MAAM;QACxB,MAAM,IAAItD,WAAW,4BAA4B,CAAC,yCAAyC,EAAEsD,YAAY3C,KAAK,CAAC,GAAG,EAAE2C,YAAYnC,KAAK,CAAC,4BAA4B,EAAEL,MAAMM,IAAI,GAAG,sFAAsF,CAAC;IAC1Q;IACA,MAAM,EAAEmC,SAAS,EAAEC,SAAS,EAAEC,SAAS,EAAEC,KAAK,EAAE,GAAGL;IACnD,MAAM,EAAE/B,KAAK,EAAEC,UAAU,EAAE,GAAGF,WAAWP;IACzC,IAAIQ,MAAMK,MAAM,KAAK,KAAKJ,WAAWI,MAAM,KAAK,GAAG,OAAO,EAAE;IAE5D,IAAIL,MAAMK,MAAM,GAAG,GAAG,MAAMM,eAAeC,MAAMC;IAEjD,MAAM,EAAEc,QAAQ,EAAEC,OAAO,EAAEC,MAAM,EAAE,GAAGb,kBAAkBhB,OAAOC;IAC/D,MAAMoC,MAAM,CAAC;oCACqB,EAAEV,SAAS;;IAE3C,EAAEM,UAAU;UACN,EAAEL,QAAQ;IAChB,EAAEM,UAAU,CAAC,EAAEC,UAAU;wCACW,CAAC;IACvC,MAAMG,OAAO,MAAM1B,KAAK2B,OAAO,CAACF;IAChC,OAAQ,MAAMC,KAAKE,GAAG,IAAIX,QAAQO;AACpC;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,0FAA0F;AAC1F,OAAO,SAASK,mBAAmB7B,IAAgB;IACjD,MAAMC,QAAuB;QAAEC,OAAO;IAAK;IAC3C,OAAO;QACL4B,OAAO,CAAClD,OAAOuC,OAASD,aAAalB,MAAMpB,OAAOuC,MAAMlB;QACxD8B,WAAW;YACT9B,MAAMC,KAAK,GAAG;QAChB;IACF;AACF"}

package/dist/esm/store/duckdb/native.d.ts

@@ -1,3 +1,2 @@
 export declare const DUCKDB_PACKAGE = "@duckdb/node-api";
-export declare function loadOrInstall<T>(importName: string, nodeModulesPath: string): Promise<T>;
 export declare function duckdbApi(): Promise<typeof import('@duckdb/node-api')>;

package/dist/esm/store/duckdb/native.js

@@ -1,87 +1,10 @@
-import { existsSync, readFileSync } from 'node:fs';
-import Module from 'node:module';
-import { dirname, join } from 'node:path';
-import { fileURLToPath, pathToFileURL } from 'node:url';
-import { SenseError } from '../../errors.js';
+import { loadOrInstall, packageNodeModules } from '../native.js';
 export const DUCKDB_PACKAGE = '@duckdb/node-api';
-// install-module-linked itself is small, but its target (@duckdb/node-api) is not: deferred the
-// same way embed/static.ts defers @huggingface/tokenizers, so requiring it stays inside the
-// already-gated branch below rather than running for every store/index.ts load.
-const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
-const _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));
-// Walks up from this file's own directory to find the real package.json, rather than a fixed
-// dot-count from __dirname (xz-compat's `../../node_modules`): `tsds test:node` runs straight
-// off src/, one directory shallower than dist/{cjs,esm}/store/duckdb sits under the built
-// package, so a hardcoded depth is right for one and wrong for the other. Works the same way
-// for a globally-installed `sense`, whose node_modules this resolves to -- never the caller's
-// cwd.
-function packageNodeModules() {
-    let dir = _dirname;
-    for(;;){
-        const candidate = join(dir, 'package.json');
-        if (existsSync(candidate)) {
-            const pkg = JSON.parse(readFileSync(candidate, 'utf8'));
-            if (pkg.name === 'sensemaking') return join(dir, 'node_modules');
-        }
-        const parent = dirname(dir);
-        if (parent === dir) throw new Error('could not locate the sensemaking package root to install @duckdb/node-api into');
-        dir = parent;
-    }
-}
-const installing = new Map();
-// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).
-// The install here is async -- the load site already awaits a dynamic import -- so a bare
-// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target
-// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,
-// keyed by specifier and target so tests exercising several installs in one process don't share
-// one slot. A failed install is not cached, so a later call retries.
-function installOnce(specifier, nodeModulesPath) {
-    const key = `${specifier}\n${nodeModulesPath}`;
-    let promise = installing.get(key);
-    if (!promise) {
-        const installModuleLinked = _require('install-module-linked').default;
-        promise = installModuleLinked(specifier, nodeModulesPath).then(()=>{}, (err)=>{
-            installing.delete(key);
-            throw err;
-        });
-        installing.set(key, promise);
-    }
-    return promise;
-}
-// Node caches a bare specifier's package resolution -- including a "not found" miss -- for the
-// life of the process, so retrying `import(importName)` right after installing sees the same
-// stale miss and falls through to a bogus index.js guess instead of the package's real entry
-// (verified against a real global install: the plain-specifier retry failed there, this did
-// not). Reading the freshly-installed package's own package.json and importing its entry file
-// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.
-function resolveEntryUrl(importName, nodeModulesPath) {
-    var _pkg_main;
-    const pkgDir = join(nodeModulesPath, ...importName.split('/'));
-    const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8'));
-    const entry = typeof pkg.exports === 'string' ? pkg.exports : (_pkg_main = pkg.main) !== null && _pkg_main !== void 0 ? _pkg_main : 'index.js';
-    return pathToFileURL(join(pkgDir, entry)).href;
-}
-// Import-then-install-then-retry, over an injectable target so the failure path (import AND
-// install both fail) can be exercised for real in tests without downloading @duckdb/node-api's
-// real ~110MB native binding. Production always passes @duckdb/node-api's own name; the thrown
-// error names @duckdb/node-api regardless, since that is what a real user needs to install.
-export async function loadOrInstall(importName, nodeModulesPath) {
-    try {
-        return await import(importName);
-    } catch  {
-        console.error(`sense: store "duckdb" needs ${DUCKDB_PACKAGE}; installing it now (one-time download, ~110MB)...`);
-        try {
-            await installOnce(importName, nodeModulesPath);
-        } catch (installErr) {
-            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" needs ${DUCKDB_PACKAGE}, and installing it automatically failed (${installErr.message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \`npm install ${DUCKDB_PACKAGE}\` yourself and try again`);
-        }
-        try {
-            return await import(resolveEntryUrl(importName, nodeModulesPath));
-        } catch (loadErr) {
-            throw new SenseError('STORE_DEPENDENCY_MISSING', `store "duckdb" installed ${DUCKDB_PACKAGE} but it could not be loaded (${loadErr.message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \`npm install ${DUCKDB_PACKAGE}\` to see the underlying error`);
-        }
-    }
-}
+const DUCKDB = {
+    store: 'duckdb',
+    pkg: DUCKDB_PACKAGE,
+    sizeHint: '~110MB'
+};
 let duckdbApiPromise;
 // The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)
 // must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh
@@ -91,6 +14,6 @@ let duckdbApiPromise;
 // problem loadOrInstall works around for its own retry). Caching the resolved module here means
 // only the first caller ever resolves the bare specifier at all.
 export function duckdbApi() {
-    if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB_PACKAGE, packageNodeModules());
+    if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall(DUCKDB, packageNodeModules());
     return duckdbApiPromise;
 }

package/dist/esm/store/duckdb/native.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { dirname, join } from 'node:path';\nimport { fileURLToPath, pathToFileURL } from 'node:url';\nimport { SenseError } from '../../errors.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\n// install-module-linked itself is small, but its target (@duckdb/node-api) is not: deferred the\n// same way embed/static.ts defers @huggingface/tokenizers, so requiring it stays inside the\n// already-gated branch below rather than running for every store/index.ts load.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\nconst _dirname = dirname(typeof __filename !== 'undefined' ? __filename : fileURLToPath(import.meta.url));\n\n// Walks up from this file's own directory to find the real package.json, rather than a fixed\n// dot-count from __dirname (xz-compat's `../../node_modules`): `tsds test:node` runs straight\n// off src/, one directory shallower than dist/{cjs,esm}/store/duckdb sits under the built\n// package, so a hardcoded depth is right for one and wrong for the other. Works the same way\n// for a globally-installed `sense`, whose node_modules this resolves to -- never the caller's\n// cwd.\nfunction packageNodeModules(): string {\n  let dir = _dirname;\n  for (;;) {\n    const candidate = join(dir, 'package.json');\n    if (existsSync(candidate)) {\n      const pkg = JSON.parse(readFileSync(candidate, 'utf8')) as { name?: string };\n      if (pkg.name === 'sensemaking') return join(dir, 'node_modules');\n    }\n    const parent = dirname(dir);\n    if (parent === dir) throw new Error('could not locate the sensemaking package root to install @duckdb/node-api into');\n    dir = parent;\n  }\n}\n\nconst installing = new Map<string, Promise<void>>();\n\n// De-dupes concurrent installs within this process (see spawn-term's loadInk for the shape).\n// The install here is async -- the load site already awaits a dynamic import -- so a bare\n// attempted flag (xz-compat's `installationAttempted`, enough for its synchronous single-target\n// install) isn't enough on its own; concurrent callers instead await the same in-flight promise,\n// keyed by specifier and target so tests exercising several installs in one process don't share\n// one slot. A failed install is not cached, so a later call retries.\nfunction installOnce(specifier: string, nodeModulesPath: string): Promise<void> {\n  const key = `${specifier}\\n${nodeModulesPath}`;\n  let promise = installing.get(key);\n  if (!promise) {\n    const installModuleLinked = (_require('install-module-linked') as { default: (specifier: string, nodeModulesPath: string) => Promise<string> }).default;\n    promise = installModuleLinked(specifier, nodeModulesPath).then(\n      () => {},\n      (err) => {\n        installing.delete(key);\n        throw err;\n      }\n    );\n    installing.set(key, promise);\n  }\n  return promise;\n}\n\n// Node caches a bare specifier's package resolution -- including a \"not found\" miss -- for the\n// life of the process, so retrying `import(importName)` right after installing sees the same\n// stale miss and falls through to a bogus index.js guess instead of the package's real entry\n// (verified against a real global install: the plain-specifier retry failed there, this did\n// not). Reading the freshly-installed package's own package.json and importing its entry file\n// by an absolute file: URL sidesteps that stale cache instead of re-resolving the bare name.\nfunction resolveEntryUrl(importName: string, nodeModulesPath: string): string {\n  const pkgDir = join(nodeModulesPath, ...importName.split('/'));\n  const pkg = JSON.parse(readFileSync(join(pkgDir, 'package.json'), 'utf8')) as { main?: string; exports?: string };\n  const entry = typeof pkg.exports === 'string' ? pkg.exports : (pkg.main ?? 'index.js');\n  return pathToFileURL(join(pkgDir, entry)).href;\n}\n\n// Import-then-install-then-retry, over an injectable target so the failure path (import AND\n// install both fail) can be exercised for real in tests without downloading @duckdb/node-api's\n// real ~110MB native binding. Production always passes @duckdb/node-api's own name; the thrown\n// error names @duckdb/node-api regardless, since that is what a real user needs to install.\nexport async function loadOrInstall<T>(importName: string, nodeModulesPath: string): Promise<T> {\n  try {\n    return (await import(importName)) as T;\n  } catch {\n    console.error(`sense: store \"duckdb\" needs ${DUCKDB_PACKAGE}; installing it now (one-time download, ~110MB)...`);\n    try {\n      await installOnce(importName, nodeModulesPath);\n    } catch (installErr) {\n      throw new SenseError(\n        'STORE_DEPENDENCY_MISSING',\n        `store \"duckdb\" needs ${DUCKDB_PACKAGE}, and installing it automatically failed (${(installErr as Error).message}); this can happen offline, in a sandboxed environment, or when node_modules is read-only or owned by another user (e.g. under a global install) -- run \\`npm install ${DUCKDB_PACKAGE}\\` yourself and try again`\n      );\n    }\n    try {\n      return (await import(resolveEntryUrl(importName, nodeModulesPath))) as T;\n    } catch (loadErr) {\n      throw new SenseError('STORE_DEPENDENCY_MISSING', `store \"duckdb\" installed ${DUCKDB_PACKAGE} but it could not be loaded (${(loadErr as Error).message}); it may not be available for this platform (${process.platform}-${process.arch}) -- run \\`npm install ${DUCKDB_PACKAGE}\\` to see the underlying error`);\n    }\n  }\n}\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)\n// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh\n// install has happened in this process, Node keeps returning that pre-install \"not found\" miss\n// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere\n// in the process fails even though the package is now genuinely on disk (the same stale-cache\n// problem loadOrInstall works around for its own retry). Caching the resolved module here means\n// only the first caller ever resolves the bare specifier at all.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB_PACKAGE, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["existsSync","readFileSync","Module","dirname","join","fileURLToPath","pathToFileURL","SenseError","DUCKDB_PACKAGE","_require","require","createRequire","url","_dirname","__filename","packageNodeModules","dir","candidate","pkg","JSON","parse","name","parent","Error","installing","Map","installOnce","specifier","nodeModulesPath","key","promise","get","installModuleLinked","default","then","err","delete","set","resolveEntryUrl","importName","pkgDir","split","entry","exports","main","href","loadOrInstall","console","error","installErr","message","loadErr","process","platform","arch","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAC1C,SAASC,aAAa,EAAEC,aAAa,QAAQ,WAAW;AACxD,SAASC,UAAU,QAAQ,kBAAkB;AAE7C,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,gGAAgG;AAChG,4FAA4F;AAC5F,gFAAgF;AAChF,MAAMC,WAAW,OAAOC,YAAY,cAAcR,OAAOS,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAC1F,MAAMG,WAAWV,QAAQ,OAAOW,eAAe,cAAcA,aAAaT,cAAc,YAAYO,GAAG;AAEvG,6FAA6F;AAC7F,8FAA8F;AAC9F,0FAA0F;AAC1F,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO;AACP,SAASG;IACP,IAAIC,MAAMH;IACV,OAAS;QACP,MAAMI,YAAYb,KAAKY,KAAK;QAC5B,IAAIhB,WAAWiB,YAAY;YACzB,MAAMC,MAAMC,KAAKC,KAAK,CAACnB,aAAagB,WAAW;YAC/C,IAAIC,IAAIG,IAAI,KAAK,eAAe,OAAOjB,KAAKY,KAAK;QACnD;QACA,MAAMM,SAASnB,QAAQa;QACvB,IAAIM,WAAWN,KAAK,MAAM,IAAIO,MAAM;QACpCP,MAAMM;IACR;AACF;AAEA,MAAME,aAAa,IAAIC;AAEvB,6FAA6F;AAC7F,0FAA0F;AAC1F,gGAAgG;AAChG,iGAAiG;AACjG,gGAAgG;AAChG,qEAAqE;AACrE,SAASC,YAAYC,SAAiB,EAAEC,eAAuB;IAC7D,MAAMC,MAAM,GAAGF,UAAU,EAAE,EAAEC,iBAAiB;IAC9C,IAAIE,UAAUN,WAAWO,GAAG,CAACF;IAC7B,IAAI,CAACC,SAAS;QACZ,MAAME,sBAAsB,AAACvB,SAAS,yBAA0GwB,OAAO;QACvJH,UAAUE,oBAAoBL,WAAWC,iBAAiBM,IAAI,CAC5D,KAAO,GACP,CAACC;YACCX,WAAWY,MAAM,CAACP;YAClB,MAAMM;QACR;QAEFX,WAAWa,GAAG,CAACR,KAAKC;IACtB;IACA,OAAOA;AACT;AAEA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,4FAA4F;AAC5F,8FAA8F;AAC9F,6FAA6F;AAC7F,SAASQ,gBAAgBC,UAAkB,EAAEX,eAAuB;QAGHV;IAF/D,MAAMsB,SAASpC,KAAKwB,oBAAoBW,WAAWE,KAAK,CAAC;IACzD,MAAMvB,MAAMC,KAAKC,KAAK,CAACnB,aAAaG,KAAKoC,QAAQ,iBAAiB;IAClE,MAAME,QAAQ,OAAOxB,IAAIyB,OAAO,KAAK,WAAWzB,IAAIyB,OAAO,IAAIzB,YAAAA,IAAI0B,IAAI,cAAR1B,uBAAAA,YAAY;IAC3E,OAAOZ,cAAcF,KAAKoC,QAAQE,QAAQG,IAAI;AAChD;AAEA,4FAA4F;AAC5F,+FAA+F;AAC/F,+FAA+F;AAC/F,4FAA4F;AAC5F,OAAO,eAAeC,cAAiBP,UAAkB,EAAEX,eAAuB;IAChF,IAAI;QACF,OAAQ,MAAM,MAAM,CAACW;IACvB,EAAE,OAAM;QACNQ,QAAQC,KAAK,CAAC,CAAC,4BAA4B,EAAExC,eAAe,kDAAkD,CAAC;QAC/G,IAAI;YACF,MAAMkB,YAAYa,YAAYX;QAChC,EAAE,OAAOqB,YAAY;YACnB,MAAM,IAAI1C,WACR,4BACA,CAAC,qBAAqB,EAAEC,eAAe,0CAA0C,EAAE,AAACyC,WAAqBC,OAAO,CAAC,sKAAsK,EAAE1C,eAAe,yBAAyB,CAAC;QAEtU;QACA,IAAI;YACF,OAAQ,MAAM,MAAM,CAAC8B,gBAAgBC,YAAYX;QACnD,EAAE,OAAOuB,SAAS;YAChB,MAAM,IAAI5C,WAAW,4BAA4B,CAAC,yBAAyB,EAAEC,eAAe,6BAA6B,EAAE,AAAC2C,QAAkBD,OAAO,CAAC,8CAA8C,EAAEE,QAAQC,QAAQ,CAAC,CAAC,EAAED,QAAQE,IAAI,CAAC,uBAAuB,EAAE9C,eAAe,8BAA8B,CAAC;QAChT;IACF;AACF;AAEA,IAAI+C;AAEJ,8FAA8F;AAC9F,6FAA6F;AAC7F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,gGAAgG;AAChG,iEAAiE;AACjE,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBT,cAAiDtC,gBAAgBO;IAC3G,OAAOwC;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/native.ts"],"sourcesContent":["import { loadOrInstall, type NativeDescriptor, packageNodeModules } from '../native.ts';\n\nexport const DUCKDB_PACKAGE = '@duckdb/node-api';\n\nconst DUCKDB: NativeDescriptor = { store: 'duckdb', pkg: DUCKDB_PACKAGE, sizeHint: '~110MB' };\n\nlet duckdbApiPromise: Promise<typeof import('@duckdb/node-api')> | undefined;\n\n// The one accessor every consumer of @duckdb/node-api (open.ts, sql-functions.ts, vectors.ts)\n// must go through, rather than each doing its own `import('@duckdb/node-api')`: once a fresh\n// install has happened in this process, Node keeps returning that pre-install \"not found\" miss\n// to any NEW bare-specifier lookup of the same path, so a second independent import elsewhere\n// in the process fails even though the package is now genuinely on disk (the same stale-cache\n// problem loadOrInstall works around for its own retry). Caching the resolved module here means\n// only the first caller ever resolves the bare specifier at all.\nexport function duckdbApi(): Promise<typeof import('@duckdb/node-api')> {\n  if (!duckdbApiPromise) duckdbApiPromise = loadOrInstall<typeof import('@duckdb/node-api')>(DUCKDB, packageNodeModules());\n  return duckdbApiPromise;\n}\n"],"names":["loadOrInstall","packageNodeModules","DUCKDB_PACKAGE","DUCKDB","store","pkg","sizeHint","duckdbApiPromise","duckdbApi"],"mappings":"AAAA,SAASA,aAAa,EAAyBC,kBAAkB,QAAQ,eAAe;AAExF,OAAO,MAAMC,iBAAiB,mBAAmB;AAEjD,MAAMC,SAA2B;IAAEC,OAAO;IAAUC,KAAKH;IAAgBI,UAAU;AAAS;AAE5F,IAAIC;AAEJ,8FAA8F;AAC9F,6FAA6F;AAC7F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,gGAAgG;AAChG,iEAAiE;AACjE,OAAO,SAASC;IACd,IAAI,CAACD,kBAAkBA,mBAAmBP,cAAiDG,QAAQF;IACnG,OAAOM;AACT"}

package/dist/esm/store/duckdb/sql-functions.js

@@ -39,7 +39,7 @@ export async function registerFunctions(conn) {
     // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars
     // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)
     // and never calls this, but hand-written raw SQL naming it still gets the real implementation
-    // rather than a silent passthrough (principle 6).
+    // rather than a silent passthrough (PRINCIPLES: no-silent-modes).
     conn.registerScalarFunction(DuckDBScalarFunction.create({
         name: 'segment',
         parameterTypes: [

package/dist/esm/store/duckdb/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional\n// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a\n// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already\n// resolved the package, possibly via an on-demand install, and a second independent bare-specifier\n// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars\n  // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)\n  // and never calls this, but hand-written raw SQL naming it still gets the real implementation\n  // rather than a silent passthrough (principle 6).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,iGAAiG;AACjG,gGAAgG;AAChG,6FAA6F;AAC7F,mGAAmG;AACnG,iGAAiG;AACjG,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,0FAA0F;IAC1F,8FAA8F;IAC9F,8FAA8F;IAC9F,kDAAkD;IAClDrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never even attempt to resolve this optional\n// peer dependency (see open.ts's connect()). Goes through native.ts's shared duckdbApi(), not a\n// bare `import('@duckdb/node-api')` of its own -- by the time this runs, open.ts has already\n// resolved the package, possibly via an on-demand install, and a second independent bare-specifier\n// lookup in the same process would hit Node's stale pre-install resolution miss (see native.ts).\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars\n  // need (see src/text/segment.ts); this store's own lexical index uses contains() instead (D1)\n  // and never calls this, but hand-written raw SQL naming it still gets the real implementation\n  // rather than a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["segmentMatch","basenameImpl","hasImpl","duckdbApi","registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","undefined","textVec","String"],"mappings":"AACA,SAASA,YAAY,QAAQ,wBAAwB;AACrD,SAASC,YAAY,EAAEC,OAAO,QAAQ,sBAAsB;AAC5D,SAASC,SAAS,QAAQ,cAAc;AAExC,iGAAiG;AACjG,gGAAgG;AAChG,6FAA6F;AAC7F,mGAAmG;AACnG,iGAAiG;AACjG,OAAO,eAAeC,kBAAkBC,IAAsB;IAC5D,MAAM,EAAEC,iBAAiB,EAAEC,oBAAoB,EAAEC,iBAAiB,EAAE,GAAG,MAAML;IAC7EE,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;YAAEL,kBAAkBK,QAAQ;SAAC;QACxEC,YAAYR,kBAAkBO,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMC,WAAWF,WAAWG,eAAe,CAAC;YAC5C,MAAMC,WAAWJ,WAAWG,eAAe,CAAC;YAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGpB,QAAQiB,SAASM,OAAO,CAACH,IAAID,SAASI,OAAO,CAACH;YACpHJ,aAAaQ,KAAK;QACpB;IACF;IAEFrB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5Cc,aAAanB,kBAAkBK,QAAQ;QACvCC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;YAC1C,MAAMU,UAAUX,WAAWG,eAAe,CAAC;YAC3C,MAAMS,YAAYZ,WAAWa,WAAW,GAAG,IAAIb,WAAWG,eAAe,CAAC,KAAK;YAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGrB,aAAa2B,QAAQH,OAAO,CAACH,IAAIO,YAAYA,UAAUJ,OAAO,CAACH,KAAKS;YAC1Ib,aAAaQ,KAAK;QACpB;IACF;IAEF,0FAA0F;IAC1F,8FAA8F;IAC9F,8FAA8F;IAC9F,kEAAkE;IAClErB,KAAKI,sBAAsB,CACzBF,qBAAqBG,MAAM,CAAC;QAC1BC,MAAM;QACNC,gBAAgB;YAACJ,kBAAkBK,QAAQ;SAAC;QAC5CC,YAAYN,kBAAkBK,QAAQ;QACtCE,cAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gBAEgDc;YAD1F,MAAMA,UAAUf,WAAWG,eAAe,CAAC;YAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGtB,aAAaiC,QAAOD,mBAAAA,QAAQP,OAAO,CAACH,gBAAhBU,8BAAAA,mBAAsB;YAChHd,aAAaQ,KAAK;QACpB;IACF;AAEJ"}

package/dist/esm/store/duckdb/store.js

@@ -1,11 +1,9 @@
 import { STORE_DIMS } from '../../embed/types.js';
 import { getColumns } from '../shared.js';
-// pending()/hasVector() are IS NULL/IS NOT NULL checks the portable Connection already
-// answers identically regardless of the vector column's type, so this store reuses sqlite's
-// (engine-neutral despite the module name) instead of duplicating them.
-import { hasVectorRow, pendingRows } from '../sqlite/vectors.js';
 import { withTransaction } from '../transaction.js';
+import { hasVectorRow, pendingRows } from '../vectors.js';
 import { storeRowToJs } from './connection.js';
+import { fieldStats } from './fieldStats.js';
 import { createLexicalIndex } from './lexical.js';
 import { reconcile } from './reconcile.js';
 import { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.js';
@@ -54,7 +52,8 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
                 return [
                     ...await getColumns(conn)
                 ];
-            }
+            },
+            fieldStats: (columns, scopeWhere)=>fieldStats(conn, columns, scopeWhere)
         },
         lexical: {
             query: lex.query
@@ -69,6 +68,10 @@ export function createStore(instance, duckdb, conn, cfg, baseDir) {
             similar: (path, opts)=>scanSimilar(duckdb, conn, STORE_DIMS, path, opts),
             hasVector: (path)=>hasVectorRow(conn, path)
         },
+        // No comparable setting to report yet.
+        async engineStatus () {
+            return {};
+        },
         raw: {
             async prepare (sql) {
                 const prepared = await duckdb.prepare(sql);