sensemaking 0.27.0 → 0.28.0

package/dist/cjs/store/duckdb/ordered-bm25.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import { createHash } from 'node:crypto';\nimport { SenseError } from '../../errors.ts';\nimport type { Connection } from '../types.ts';\n\n// This is a connection-local TEMP macro. The contract hash covers the native fts definition,\n// DuckDB version, and the reviewed parameter defaults captured in the retained diagnostic.\nexport const ORDERED_BM25_MACRO = 'sense_ordered_match_bm25';\nconst SUPPORTED_DUCKDB_VERSION = 'v1.5.5';\nconst SUPPORTED_CONTRACT_SHA256 = '4476d543e43e7f8c89751cac0659b64e09d906cd3fefb1bfcaf4d15d3b1d9e00';\nconst EXPECTED_PARAMETERS = ['docname', 'query_string', 'fields', 'k', 'b', 'conjunctive'];\nconst EXPECTED_DEFAULTS = { fields: 'NULL', k: '1.2', b: '0.75', conjunctive: 'false' } as const;\nconst EXPECTED_SIGNATURE = `docname, query_string, fields := ${EXPECTED_DEFAULTS.fields}, k := ${EXPECTED_DEFAULTS.k}, b := ${EXPECTED_DEFAULTS.b}, conjunctive := ${EXPECTED_DEFAULTS.conjunctive}`;\nconst UNORDERED_SUM = 'sum(subscore)';\n\ninterface NativeMacroRow {\n  function_type: string;\n  parameters: unknown;\n  parameter_types: unknown;\n  macro_definition: string;\n}\n\ninterface NativeMacroContract {\n  version: string;\n  macro: NativeMacroRow;\n}\n\nfunction unsupported(message: string): SenseError {\n  return new SenseError('STORE_CAPABILITY_MISSING', `store \"duckdb\" cannot install its deterministic native BM25 adapter: ${message}; update the DuckDB adapter before using lexical search`);\n}\n\nfunction contractSha256(version: string, definition: string): string {\n  return createHash('sha256').update(`${version}\\0${EXPECTED_SIGNATURE}\\0${definition}`).digest('hex');\n}\n\nexport function validateNativeMacroContract({ version, macro }: NativeMacroContract): string {\n  if (version !== SUPPORTED_DUCKDB_VERSION) throw unsupported(`DuckDB ${version} is unsupported (validated version is ${SUPPORTED_DUCKDB_VERSION})`);\n  if (macro.function_type !== 'macro') throw unsupported(`match_bm25 is ${macro.function_type}, not a macro`);\n  if (JSON.stringify(macro.parameters) !== JSON.stringify(EXPECTED_PARAMETERS)) throw unsupported(`match_bm25 parameters changed (${JSON.stringify(macro.parameters)})`);\n  if (!Array.isArray(macro.parameter_types) || macro.parameter_types.some((type) => type !== null)) throw unsupported('match_bm25 parameter types changed');\n  if (contractSha256(version, macro.macro_definition) !== SUPPORTED_CONTRACT_SHA256) throw unsupported('match_bm25 definition or reviewed parameter-default contract changed');\n  if (macro.macro_definition.split(UNORDERED_SUM).length !== 2) throw unsupported('match_bm25 does not contain exactly one validated unordered subscore sum');\n  return macro.macro_definition.replace(UNORDERED_SUM, 'sum(subscore ORDER BY subscore)');\n}\n\nexport async function ensureOrderedBm25(conn: Connection, state: { orderedMacroReady: boolean }): Promise<void> {\n  if (state.orderedMacroReady) return;\n  const versionStatement = await conn.prepare('SELECT version() AS version');\n  const versionRow = (await versionStatement.get()) as { version: string };\n  const macroStatement = await conn.prepare(`\n    SELECT function_type, parameters, parameter_types, macro_definition\n    FROM duckdb_functions()\n    WHERE schema_name = 'fts_main_content' AND function_name = 'match_bm25'\n  `);\n  const macro = (await macroStatement.get()) as NativeMacroRow | undefined;\n  if (!macro) throw unsupported('fts_main_content.match_bm25 is missing');\n  const orderedDefinition = validateNativeMacroContract({ version: versionRow.version, macro });\n  await conn.exec(`CREATE OR REPLACE TEMP MACRO ${ORDERED_BM25_MACRO}(${EXPECTED_SIGNATURE}) AS ${orderedDefinition}`);\n  state.orderedMacroReady = true;\n}\n"],"names":["ORDERED_BM25_MACRO","ensureOrderedBm25","validateNativeMacroContract","SUPPORTED_DUCKDB_VERSION","SUPPORTED_CONTRACT_SHA256","EXPECTED_PARAMETERS","EXPECTED_DEFAULTS","fields","k","b","conjunctive","EXPECTED_SIGNATURE","UNORDERED_SUM","unsupported","message","SenseError","contractSha256","version","definition","createHash","update","digest","macro","function_type","JSON","stringify","parameters","Array","isArray","parameter_types","some","type","macro_definition","split","length","replace","conn","state","versionStatement","versionRow","macroStatement","orderedDefinition","orderedMacroReady","prepare","get","exec"],"mappings":";;;;;;;;;;;QAMaA;eAAAA;;QAsCSC;eAAAA;;QAVNC;eAAAA;;;0BAlCW;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAKpB,IAAMF,qBAAqB;AAClC,IAAMG,2BAA2B;AACjC,IAAMC,4BAA4B;AAClC,IAAMC,sBAAsB;IAAC;IAAW;IAAgB;IAAU;IAAK;IAAK;CAAc;AAC1F,IAAMC,oBAAoB;IAAEC,QAAQ;IAAQC,GAAG;IAAOC,GAAG;IAAQC,aAAa;AAAQ;AACtF,IAAMC,qBAAqB,AAAC,oCAAqEL,OAAlCA,kBAAkBC,MAAM,EAAC,WAAsCD,OAA7BA,kBAAkBE,CAAC,EAAC,WAAgDF,OAAvCA,kBAAkBG,CAAC,EAAC,qBAAiD,OAA9BH,kBAAkBI,WAAW;AAClM,IAAME,gBAAgB;AActB,SAASC,YAAYC,OAAe;IAClC,OAAO,IAAIC,oBAAU,CAAC,4BAA4B,AAAC,wEAA+E,OAARD,SAAQ;AACpI;AAEA,SAASE,eAAeC,OAAe,EAAEC,UAAkB;IACzD,OAAOC,IAAAA,sBAAU,EAAC,UAAUC,MAAM,CAAC,AAAC,GAAcT,OAAZM,SAAQ,MAA2BC,OAAvBP,oBAAmB,MAAe,OAAXO,aAAcG,MAAM,CAAC;AAChG;AAEO,SAASnB,4BAA4B,KAAuC;QAArCe,UAAF,MAAEA,SAASK,QAAX,MAAWA;IACrD,IAAIL,YAAYd,0BAA0B,MAAMU,YAAY,AAAC,UAAyDV,OAAhDc,SAAQ,0CAAiE,OAAzBd,0BAAyB;IAC/I,IAAImB,MAAMC,aAAa,KAAK,SAAS,MAAMV,YAAY,AAAC,iBAAoC,OAApBS,MAAMC,aAAa,EAAC;IAC5F,IAAIC,KAAKC,SAAS,CAACH,MAAMI,UAAU,MAAMF,KAAKC,SAAS,CAACpB,sBAAsB,MAAMQ,YAAY,AAAC,kCAAkE,OAAjCW,KAAKC,SAAS,CAACH,MAAMI,UAAU,GAAE;IACnK,IAAI,CAACC,MAAMC,OAAO,CAACN,MAAMO,eAAe,KAAKP,MAAMO,eAAe,CAACC,IAAI,CAAC,SAACC;eAASA,SAAS;QAAO,MAAMlB,YAAY;IACpH,IAAIG,eAAeC,SAASK,MAAMU,gBAAgB,MAAM5B,2BAA2B,MAAMS,YAAY;IACrG,IAAIS,MAAMU,gBAAgB,CAACC,KAAK,CAACrB,eAAesB,MAAM,KAAK,GAAG,MAAMrB,YAAY;IAChF,OAAOS,MAAMU,gBAAgB,CAACG,OAAO,CAACvB,eAAe;AACvD;AAEO,SAAeX,kBAAkBmC,IAAgB,EAAEC,KAAqC;;YAEvFC,kBACAC,YACAC,gBAKAlB,OAEAmB;;;;oBAVN,IAAIJ,MAAMK,iBAAiB,EAAE;;;oBACJ;;wBAAMN,KAAKO,OAAO,CAAC;;;oBAAtCL,mBAAmB;oBACL;;wBAAMA,iBAAiBM,GAAG;;;oBAAxCL,aAAc;oBACG;;wBAAMH,KAAKO,OAAO,CAAC;;;oBAApCH,iBAAiB;oBAKR;;wBAAMA,eAAeI,GAAG;;;oBAAjCtB,QAAS;oBACf,IAAI,CAACA,OAAO,MAAMT,YAAY;oBACxB4B,oBAAoBvC,4BAA4B;wBAAEe,SAASsB,WAAWtB,OAAO;wBAAEK,OAAAA;oBAAM;oBAC3F;;wBAAMc,KAAKS,IAAI,CAAC,AAAC,gCAAqDlC,OAAtBX,oBAAmB,KAA6ByC,OAA1B9B,oBAAmB,SAAyB,OAAlB8B;;;oBAAhG;oBACAJ,MAAMK,iBAAiB,GAAG;;;;;;IAC5B"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/ordered-bm25.ts"],"sourcesContent":["import type { Connection } from '../types.ts';\nimport type { DuckdbFtsBuilders } from './engine.ts';\n\n// Keep deduplication native, but return original tokens so native scoring stems once.\nexport async function nativeQueryTerms(conn: Connection, fts: DuckdbFtsBuilders, query: string): Promise<string[]> {\n  const stmt = await conn.prepare(fts.queryTerms());\n  const rows = (await stmt.all(query)) as Array<{ term: string }>;\n  return rows.map(({ term }) => term);\n}\n\n// Fixed parameters avoid a correlated document-by-term intermediate relation in DuckDB.\n// Only the number of terms shapes this authored SQL; query values are always bound.\nexport function orderedBm25Sql(fts: DuckdbFtsBuilders, field: 'title' | 'summary' | 'text', termCount: number): string {\n  if (termCount === 0) return 'NULL';\n  const contributions = Array.from({ length: termCount }, () => fts.termScore(field));\n  return `(SELECT sum(score ORDER BY score) FROM (${contributions.join(' UNION ALL ')}))`;\n}\n"],"names":["nativeQueryTerms","orderedBm25Sql","conn","fts","query","stmt","rows","prepare","queryTerms","all","map","term","field","termCount","contributions","Array","from","length","termScore","join"],"mappings":";;;;;;;;;;;QAIsBA;eAAAA;;QAQNC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AART,SAAeD,iBAAiBE,IAAgB,EAAEC,GAAsB,EAAEC,KAAa;;YACtFC,MACAC;;;;oBADO;;wBAAMJ,KAAKK,OAAO,CAACJ,IAAIK,UAAU;;;oBAAxCH,OAAO;oBACC;;wBAAMA,KAAKI,GAAG,CAACL;;;oBAAvBE,OAAQ;oBACd;;wBAAOA,KAAKI,GAAG,CAAC;gCAAGC,aAAAA;mCAAWA;;;;;IAChC;;AAIO,SAASV,eAAeE,GAAsB,EAAES,KAAmC,EAAEC,SAAiB;IAC3G,IAAIA,cAAc,GAAG,OAAO;IAC5B,IAAMC,gBAAgBC,MAAMC,IAAI,CAAC;QAAEC,QAAQJ;IAAU,GAAG;eAAMV,IAAIe,SAAS,CAACN;;IAC5E,OAAO,AAAC,2CAA4E,OAAlCE,cAAcK,IAAI,CAAC,gBAAe;AACtF"}

package/dist/cjs/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so changed rows are maintained\n// here; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n// Bound the extra persisted text held for exact comparison, even during a full reparse.\nconst CONTENT_LOOKUP_CHUNK = 128;\n\ninterface ContentRow {\n  path: string;\n  title: string | null;\n  summary: string | null;\n  text: string | null;\n}\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\nconst FRONTMATTER_UPDATE_CHUNK = 128;\n\nasync function withNativeStatement<T>(native: DuckDBConnection, sql: string, fn: (stmt: DuckDBPreparedStatement) => Promise<T>): Promise<T> {\n  const stmt = await native.prepare(sql);\n  const errors: unknown[] = [];\n  let result!: T;\n  try {\n    result = await fn(stmt);\n  } catch (err) {\n    errors.push(err);\n  }\n  try {\n    stmt.destroySync();\n  } catch (err) {\n    errors.push(err);\n  }\n  if (errors.length === 1) throw errors[0];\n  if (errors.length > 1) throw new AggregateError(errors, 'frontmatter statement and cleanup failed');\n  return result;\n}\n\n// Wide writes amplify native commit allocations even for one changed stamp.\n// Skip only known, exactly equal values; uncertain representations still write.\nasync function updateFrontmatter(conn: Connection, writableColumns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const native = (conn as DuckdbConnection).duckdb;\n  const { DuckDBVariantValue, DuckDBTypeId } = await duckdbApi();\n  const physical = await withNativeStatement(native, 'PRAGMA table_info(frontmatter)', async (stmt) => {\n    const reader = await stmt.runAndReadAll();\n    return reader.getRowObjectsJS() as Array<{ name: string; type: string }>;\n  });\n  const writable = new Map(writableColumns.map((name, i) => [name, i]));\n  const pathIndex = writable.get('path');\n  if (pathIndex === undefined) throw new Error('frontmatter update requires a writable path column');\n  const columnTypes = new Map(physical.map((column) => [column.name, column.type]));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')})`;\n  const sameValue = (old: DuckDBValue, value: unknown, column: string): boolean => {\n    if (value === null || value === undefined) return old === null;\n    if (columnTypes.get(column) !== 'VARIANT') return Object.is(old, value);\n    if (!(old instanceof DuckDBVariantValue)) return false;\n    const typeId = typeof value === 'string' ? DuckDBTypeId.VARCHAR : typeof value === 'bigint' ? DuckDBTypeId.HUGEINT : typeof value === 'number' && !Number.isInteger(value) ? DuckDBTypeId.DOUBLE : undefined;\n    return typeId !== undefined && old.type?.typeId === typeId && Object.is(old.value, value);\n  };\n  for (let i = 0; i < rows.length; i += FRONTMATTER_UPDATE_CHUNK) {\n    const chunk = rows.slice(i, i + FRONTMATTER_UPDATE_CHUNK);\n    const paths = chunk.map((row) => row[pathIndex] as string);\n    const placeholders = paths.map(() => '?').join(', ');\n    const selected = writableColumns.map(quoteIdent).join(', ');\n    const existing = await withNativeStatement(native, `SELECT ${selected} FROM frontmatter WHERE \"path\" IN (${placeholders})`, async (stmt) => {\n      stmt.bind(paths);\n      const reader = await stmt.runAndReadAll();\n      // Native wrappers distinguish numeric tags and SQL NULL from VARIANT null.\n      return new Map(reader.getRowObjects().map((row) => [row.path as string, row]));\n    });\n    const plans = chunk.map((row) => {\n      const old = existing.get(row[pathIndex] as string);\n      const changed = old ? writableColumns.flatMap((column, index) => (sameValue(old[column], row[index], column) ? [] : [index])) : [];\n      return { row, missing: old === undefined, changed, signature: changed.join(',') };\n    });\n    for (let position = 0; position < plans.length; ) {\n      const plan = plans[position];\n      if (!plan.missing && plan.changed.length === 0) {\n        ++position;\n        continue;\n      }\n      let end = position + 1;\n      while (end < plans.length && plans[end].missing === plan.missing && plans[end].signature === plan.signature) ++end;\n      if (plan.missing) {\n        await appendRows(\n          conn,\n          'frontmatter',\n          writableColumns,\n          insertSql,\n          plans.slice(position, end).map((repair) => repair.row)\n        );\n      } else {\n        const assignments = plan.changed.map((index) => `${quoteIdent(writableColumns[index])} = ?`).join(', ');\n        await withNativeStatement(native, `UPDATE frontmatter SET ${assignments} WHERE \"path\" = ?`, async (stmt) => {\n          for (let rowIndex = position; rowIndex < end; ++rowIndex) {\n            const row = plans[rowIndex].row;\n            stmt.bind([...plan.changed.map((index) => row[index] ?? null), row[pathIndex]] as DuckDBValue[]);\n            await stmt.run();\n          }\n        });\n      }\n      position = end;\n    }\n  }\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const added = new Set(delta.added);\n  const reparsed = docs.filter((doc) => !added.has(doc.relPath));\n  const changed = new Set(delta.added);\n  const removed: string[] = [];\n  for (let i = 0; i < reparsed.length; i += CONTENT_LOOKUP_CHUNK) {\n    const chunk = reparsed.slice(i, i + CONTENT_LOOKUP_CHUNK);\n    const stmt = await conn.prepare(`SELECT \"path\", title, summary, text FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n    const rows = (await stmt.all(...chunk.map((doc) => doc.relPath))) as ContentRow[];\n    const existing = new Map(rows.map((row) => [row.path, row]));\n    for (const doc of chunk) {\n      const row = existing.get(doc.relPath);\n      if (row && row.title === doc.search.title && row.summary === doc.search.summary && row.text === doc.search.text) continue;\n      changed.add(doc.relPath);\n      if (row) removed.push(doc.relPath);\n    }\n  }\n\n  const docPaths = new Set(docs.map((doc) => doc.relPath));\n  const vanished = touched.filter((path) => !docPaths.has(path));\n  for (let i = 0; i < vanished.length; i += CONTENT_LOOKUP_CHUNK) {\n    const chunk = vanished.slice(i, i + CONTENT_LOOKUP_CHUNK);\n    const stmt = await conn.prepare(`SELECT \"path\" FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n    const rows = (await stmt.all(...chunk)) as Array<{ path: string }>;\n    removed.push(...rows.map((row) => row.path));\n  }\n\n  if (removed.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      removed.map((p) => [p])\n    );\n  // Keep document order and the existing insert/error semantics for added paths.\n  const changedDocs = docs.filter((doc) => changed.has(doc.relPath));\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, changedDocs.map(contentRow));\n  // Commit the stale mark with actual content changes. Identical rows leave prior staleness\n  // untouched, and FTS preparation still owns missing-index recovery.\n  if (removed.length > 0 || changedDocs.length > 0) await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  updateFrontmatter,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","CONTENT_LOOKUP_CHUNK","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","FRONTMATTER_UPDATE_CHUNK","withNativeStatement","native","sql","fn","stmt","errors","result","err","prepare","push","destroySync","length","AggregateError","updateFrontmatter","conn","writableColumns","rows","DuckDBVariantValue","DuckDBTypeId","physical","writable","pathIndex","columnTypes","insertSql","sameValue","i","chunk","paths","placeholders","selected","existing","plans","position","plan","end","assignments","missing","changed","signature","appendRows","slice","repair","row","index","rowIndex","bind","run","reader","runAndReadAll","Map","getRowObjects","path","old","get","flatMap","column","undefined","duckdb","duckdbApi","getRowObjectsJS","name","Error","type","value","Object","is","typeId","VARCHAR","HUGEINT","Number","isInteger","DOUBLE","reconcileContent","touched","docs","delta","added","reparsed","removed","docPaths","vanished","changedDocs","Set","filter","has","all","add","runBatch","p","markContentStale","addColumns","names","exec","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAgLaA;;;eAAAA;;;wBA/Kc;wBAGY;yBAGN;wBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,qFAAqF;AACrF,kGAAkG;AAClG,iFAAiF;AACjF,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAC9F,wFAAwF;AACxF,IAAMC,uBAAuB;AAS7B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAChC,IAAMC,2BAA2B;AAEjC,SAAeC,oBAAuBC,MAAwB,EAAEC,GAAW,EAAEC,EAAiD;;YACtHC,MACAC,QACFC,QAGKC;;;;oBALI;;wBAAMN,OAAOO,OAAO,CAACN;;;oBAA5BE,OAAO;oBACPC;;;;;;;;;oBAGK;;wBAAMF,GAAGC;;;oBAAlBE,SAAS;;;;;;oBACFC;oBACPF,OAAOI,IAAI,CAACF;;;;;;oBAEd,IAAI;wBACFH,KAAKM,WAAW;oBAClB,EAAE,OAAOH,KAAK;wBACZF,OAAOI,IAAI,CAACF;oBACd;oBACA,IAAIF,OAAOM,MAAM,KAAK,GAAG,MAAMN,MAAM,CAAC,EAAE;oBACxC,IAAIA,OAAOM,MAAM,GAAG,GAAG,MAAM,IAAIC,eAAeP,QAAQ;oBACxD;;wBAAOC;;;;IACT;;AAEA,4EAA4E;AAC5E,gFAAgF;AAChF,SAAeO,kBAAkBC,IAAgB,EAAEC,eAAyB,EAAEC,IAAiB;;mBAEvFf,QACuC,MAArCgB,oBAAoBC,cACtBC,UAIAC,UACAC,WAEAC,aACAC,WACAC,WAOGC;;;;;mCACDC,OACAC,OACAC,cACAC,UACAC,UAMAC,OAKGC;;;;;4CACDC,MAKFC,KAWIC;;;;oDAhBFF,OAAOF,KAAK,CAACC,UAAS;oDAC5B,IAAI,CAACC,KAAKG,OAAO,IAAIH,KAAKI,OAAO,CAAC1B,MAAM,KAAK,GAAG;wDAC9C,EAAEqB;wDACF;;mFAAA;;oDACF;oDACIE,MAAMF,YAAW;oDACrB,MAAOE,MAAMH,MAAMpB,MAAM,IAAIoB,KAAK,CAACG,IAAI,CAACE,OAAO,KAAKH,KAAKG,OAAO,IAAIL,KAAK,CAACG,IAAI,CAACI,SAAS,KAAKL,KAAKK,SAAS,CAAE,EAAEJ;yDAC3GD,KAAKG,OAAO,EAAZH;;;;oDACF;;wDAAMM,IAAAA,oBAAU,EACdzB,MACA,eACAC,iBACAQ,WACAQ,MAAMS,KAAK,CAACR,WAAUE,KAAK/C,GAAG,CAAC,SAACsD;mEAAWA,OAAOC,GAAG;;;;oDALvD;;;;;;oDAQMP,cAAcF,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;+DAAU,AAAC,GAAqC,OAAnCvD,IAAAA,oBAAU,EAAC2B,eAAe,CAAC4B,MAAM,GAAE;uDAAOtD,IAAI,CAAC;oDAClG;;wDAAMW,oBAAoBC,QAAQ,AAAC,0BAAqC,OAAZkC,aAAY,sBAAoB,SAAO/B;;2EACxFwC;;;;;oFACDF;;;;4FAAAA,MAAMX,KAAK,CAACa,SAAS,CAACF,GAAG;4FAC/BtC,KAAKyC,IAAI,CAAC,AAAC,qBAAGZ,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;oGAAUD;wGAAAA,aAAAA,GAAG,CAACC,MAAM,cAAVD,wBAAAA,aAAc;;gGAAOA,GAAG,CAACrB,UAAU;;4FAC7E;;gGAAMjB,KAAK0C,GAAG;;;4FAAd;;;;;;4EACF;4EAJSF,WAAWZ;;;iFAAUY,CAAAA,WAAWV,GAAE;;;;;;;;;;;;4EAAG,EAAEU;;;;;;;;;;;4DAKlD;;;;oDANA;;;oDAQFZ,YAAWE;;;;;;;oCACb;oCA1CMR,QAAQV,KAAKwB,KAAK,CAACf,GAAGA,IAAI1B;oCAC1B4B,QAAQD,MAAMvC,GAAG,CAAC,SAACuD;+CAAQA,GAAG,CAACrB,UAAU;;oCACzCO,eAAeD,MAAMxC,GAAG,CAAC;+CAAM;uCAAKE,IAAI,CAAC;oCACzCwC,WAAWd,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC;oCACrC;;wCAAMW,oBAAoBC,QAAQ,AAAC,UAAuD2B,OAA9CC,UAAS,uCAAkD,OAAbD,cAAa,MAAI,SAAOxB;;oDAE3H2C;;;;4DADN3C,KAAKyC,IAAI,CAAClB;4DACK;;gEAAMvB,KAAK4C,aAAa;;;4DAAjCD,SAAS;4DACf,2EAA2E;4DAC3E;;gEAAO,IAAIE,IAAIF,OAAOG,aAAa,GAAG/D,GAAG,CAAC,SAACuD;2EAAQ;wEAACA,IAAIS,IAAI;wEAAYT;qEAAI;;;;;4CAC9E;;;;oCALMZ,WAAW;oCAMXC,QAAQL,MAAMvC,GAAG,CAAC,SAACuD;wCACvB,IAAMU,MAAMtB,SAASuB,GAAG,CAACX,GAAG,CAACrB,UAAU;wCACvC,IAAMgB,UAAUe,MAAMrC,gBAAgBuC,OAAO,CAAC,SAACC,QAAQZ;mDAAWnB,UAAU4B,GAAG,CAACG,OAAO,EAAEb,GAAG,CAACC,MAAM,EAAEY,UAAU,EAAE,GAAG;gDAACZ;6CAAM;6CAAK,EAAE;wCAClI,OAAO;4CAAED,KAAAA;4CAAKN,SAASgB,QAAQI;4CAAWnB,SAAAA;4CAASC,WAAWD,QAAQhD,IAAI,CAAC;wCAAK;oCAClF;oCACS2C,WAAW;;;yCAAGA,CAAAA,WAAWD,MAAMpB,MAAM,AAAD;;;;;;;;;;;;;;;;;;;;;;oBA4B/C;oBA/DA,IAAIK,KAAKL,MAAM,KAAK,GAAG;;;oBACjBV,SAAS,AAACa,KAA0B2C,MAAM;oBACH;;wBAAMC,IAAAA,mBAAS;;;oBAAf,OAAA,eAArCzC,qBAAqC,KAArCA,oBAAoBC,eAAiB,KAAjBA;oBACX;;wBAAMlB,oBAAoBC,QAAQ,kCAAkC,SAAOG;;oCACpF2C;;;;4CAAS;;gDAAM3C,KAAK4C,aAAa;;;4CAAjCD,SAAS;4CACf;;gDAAOA,OAAOY,eAAe;;;;4BAC/B;;;;oBAHMxC,WAAW;oBAIXC,WAAW,IAAI6B,IAAIlC,gBAAgB5B,GAAG,CAAC,SAACyE,MAAMnC;+BAAM;4BAACmC;4BAAMnC;yBAAE;;oBAC7DJ,YAAYD,SAASiC,GAAG,CAAC;oBAC/B,IAAIhC,cAAcmC,WAAW,MAAM,IAAIK,MAAM;oBACvCvC,cAAc,IAAI2B,IAAI9B,SAAShC,GAAG,CAAC,SAACoE;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL,OAAOO,IAAI;yBAAC;;oBACzEvC,YAAY,AAAC,4BAAkFR,OAAvDA,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAAsD,OAA1C0B,gBAAgB5B,GAAG,CAAC;+BAAM;uBAAKE,IAAI,CAAC,OAAM;oBACzImC,YAAY,mBAAC4B,KAAkBW,OAAgBR;4BAKpBH;wBAJ/B,IAAIW,UAAU,QAAQA,UAAUP,WAAW,OAAOJ,QAAQ;wBAC1D,IAAI9B,YAAY+B,GAAG,CAACE,YAAY,WAAW,OAAOS,OAAOC,EAAE,CAACb,KAAKW;wBACjE,IAAI,CAAEX,AAAG,YAAHA,KAAenC,qBAAqB,OAAO;wBACjD,IAAMiD,SAAS,OAAOH,UAAU,WAAW7C,aAAaiD,OAAO,GAAG,CAAA,OAAOJ,sCAAP,SAAOA,MAAI,MAAM,WAAW7C,aAAakD,OAAO,GAAG,OAAOL,UAAU,YAAY,CAACM,OAAOC,SAAS,CAACP,SAAS7C,aAAaqD,MAAM,GAAGf;wBACnM,OAAOU,WAAWV,aAAaJ,EAAAA,YAAAA,IAAIU,IAAI,cAARV,gCAAAA,UAAUc,MAAM,MAAKA,UAAUF,OAAOC,EAAE,CAACb,IAAIW,KAAK,EAAEA;oBACrF;oBACStC,IAAI;;;yBAAGA,CAAAA,IAAIT,KAAKL,MAAM,AAAD;;;;;;;;;;;;oBAAGc,KAAK1B;;;;;;;;;;;IA6CxC;;AAEA,SAAeyE,iBAAiB1D,IAAgB,EAAE2D,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OACAC,UACAxC,SACAyC,SACGrD,GAGarB,OAFdsB,OACAtB,MACAY,MACAc,UACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMtC,KACHkD,KAOJqC,UACAC,UACGvD,IAGarB,QACpB0E,UAHMpD,QACAtB,OACAY,OAUFiE;;;;oBAhCAL,QAAQ,IAAIM,IAAIP,MAAMC,KAAK;oBAC3BC,WAAWH,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ,CAACoF,MAAMQ,GAAG,CAAC5F,IAAIC,OAAO;;oBACtD4C,UAAU,IAAI6C,IAAIP,MAAMC,KAAK;oBAC7BE;oBACGrD,IAAI;;;yBAAGA,CAAAA,IAAIoD,SAASlE,MAAM,AAAD;;;;oBAC1Be,QAAQmD,SAASrC,KAAK,CAACf,GAAGA,IAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,qEAAoG,OAAhCkB,MAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAA/He,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKiF,GAAG,OAARjF,OAAS,qBAAGsB,MAAMvC,GAAG,CAAC,SAACK;mCAAQA,IAAIC,OAAO;;;;oBAAxDuB,OAAQ;oBACRc,WAAW,IAAImB,IAAIjC,KAAK7B,GAAG,CAAC,SAACuD;+BAAQ;4BAACA,IAAIS,IAAI;4BAAET;yBAAI;;oBACrD,kCAAA,2BAAA;;wBAAL,IAAK,YAAahB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdlC,MAAN;4BACGkD,MAAMZ,SAASuB,GAAG,CAAC7D,IAAIC,OAAO;4BACpC,IAAIiD,OAAOA,IAAI/C,KAAK,KAAKH,IAAIE,MAAM,CAACC,KAAK,IAAI+C,IAAI9C,OAAO,KAAKJ,IAAIE,MAAM,CAACE,OAAO,IAAI8C,IAAI7C,IAAI,KAAKL,IAAIE,MAAM,CAACG,IAAI,EAAE;4BACjHwC,QAAQiD,GAAG,CAAC9F,IAAIC,OAAO;4BACvB,IAAIiD,KAAKoC,QAAQrE,IAAI,CAACjB,IAAIC,OAAO;wBACnC;;wBALK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAL8BgC,KAAKnC;;;;;;oBAapCyF,WAAW,IAAIG,IAAIR,KAAKvF,GAAG,CAAC,SAACK;+BAAQA,IAAIC,OAAO;;oBAChDuF,WAAWP,QAAQU,MAAM,CAAC,SAAChC;+BAAS,CAAC4B,SAASK,GAAG,CAACjC;;oBAC/C1B,KAAI;;;yBAAGA,CAAAA,KAAIuD,SAASrE,MAAM,AAAD;;;;oBAC1Be,SAAQsD,SAASxC,KAAK,CAACf,IAAGA,KAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,+CAA8E,OAAhCkB,OAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAAzGe,QAAO;oBACC;;wBAAMA,CAAAA,SAAAA,OAAKiF,GAAG,OAARjF,QAAS,qBAAGsB;;;oBAA1BV,QAAQ;oBACd8D,CAAAA,WAAAA,SAAQrE,IAAI,OAAZqE,UAAa,qBAAG9D,MAAK7B,GAAG,CAAC,SAACuD;+BAAQA,IAAIS,IAAI;;;;oBAJP1B,MAAKnC;;;;;;yBAOtCwF,CAAAA,QAAQnE,MAAM,GAAG,CAAA,GAAjBmE;;;;oBACF;;wBAAMhE,KAAKyE,QAAQ,CACjB,wCACAT,QAAQ3F,GAAG,CAAC,SAACqG;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,+EAA+E;oBACzEP,cAAcP,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ6C,QAAQ+C,GAAG,CAAC5F,IAAIC,OAAO;;oBAChE;;wBAAM8C,IAAAA,oBAAU,EAACzB,MAAM,WAAW7B,iBAAiBC,oBAAoB+F,YAAY9F,GAAG,CAACI;;;oBAAvF;yBAGIuF,CAAAA,QAAQnE,MAAM,GAAG,KAAKsE,YAAYtE,MAAM,GAAG,CAAA,GAA3CmE;;;;oBAA8C;;wBAAMW,IAAAA,2BAAgB,EAAC3E;;;oBAAvB;;;;;;;;IACpD;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAe4E,WAAW5E,IAAgB,EAAE6E,KAAe;;;;;oBACzD,IAAIA,MAAMhF,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMG,KAAK8E,IAAI,CAACD,MAAMxG,GAAG,CAAC,SAACyE;mCAAS,AAAC,sCAAsD,OAAjBxE,IAAAA,oBAAU,EAACwE,OAAM;2BAAWvE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C6G,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjG,yBAAyB;YACnC,MAAM,IAAIkG,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElG,OAAtDiG,OAAM,kDAAwE,OAAxBjG,yBAAwB;QAC/I;IACF;IACA4F,YAAAA;IACA7E,mBAAAA;IACA2D,kBAAAA;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { appendRows, quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so changed rows are maintained\n// here; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst CONTENT_COLUMNS = ['path', 'title', 'summary', 'text'];\nconst INSERT_CONTENT_SQL = `INSERT INTO content (${CONTENT_COLUMNS.map(quoteIdent).join(', ')}) VALUES (?, ?, ?, ?)`;\n// Bound the extra persisted text held for exact comparison, even during a full reparse.\nconst CONTENT_LOOKUP_CHUNK = 128;\n\ninterface ContentRow {\n  path: string;\n  title: string | null;\n  summary: string | null;\n  text: string | null;\n}\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\nconst FRONTMATTER_UPDATE_CHUNK = 128;\n\nasync function withNativeStatement<T>(native: DuckDBConnection, sql: string, fn: (stmt: DuckDBPreparedStatement) => Promise<T>): Promise<T> {\n  const stmt = await native.prepare(sql);\n  const errors: unknown[] = [];\n  let result!: T;\n  try {\n    result = await fn(stmt);\n  } catch (err) {\n    errors.push(err);\n  }\n  try {\n    stmt.destroySync();\n  } catch (err) {\n    errors.push(err);\n  }\n  if (errors.length === 1) throw errors[0];\n  if (errors.length > 1) throw new AggregateError(errors, 'frontmatter statement and cleanup failed');\n  return result;\n}\n\n// Wide writes amplify native commit allocations even for one changed stamp.\n// Skip only known, exactly equal values; uncertain representations still write.\nasync function updateFrontmatter(conn: Connection, writableColumns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const native = (conn as DuckdbConnection).duckdb;\n  const { DuckDBVariantValue, DuckDBTypeId } = await duckdbApi();\n  const physical = await withNativeStatement(native, 'PRAGMA table_info(frontmatter)', async (stmt) => {\n    const reader = await stmt.runAndReadAll();\n    return reader.getRowObjectsJS() as Array<{ name: string; type: string }>;\n  });\n  const writable = new Map(writableColumns.map((name, i) => [name, i]));\n  const pathIndex = writable.get('path');\n  if (pathIndex === undefined) throw new Error('frontmatter update requires a writable path column');\n  const columnTypes = new Map(physical.map((column) => [column.name, column.type]));\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')})`;\n  const sameValue = (old: DuckDBValue, value: unknown, column: string): boolean => {\n    if (value === null || value === undefined) return old === null;\n    if (columnTypes.get(column) !== 'VARIANT') return Object.is(old, value);\n    if (!(old instanceof DuckDBVariantValue)) return false;\n    const typeId = typeof value === 'string' ? DuckDBTypeId.VARCHAR : typeof value === 'bigint' ? DuckDBTypeId.HUGEINT : typeof value === 'number' && !Number.isInteger(value) ? DuckDBTypeId.DOUBLE : undefined;\n    return typeId !== undefined && old.type?.typeId === typeId && Object.is(old.value, value);\n  };\n  for (let i = 0; i < rows.length; i += FRONTMATTER_UPDATE_CHUNK) {\n    const chunk = rows.slice(i, i + FRONTMATTER_UPDATE_CHUNK);\n    const paths = chunk.map((row) => row[pathIndex] as string);\n    const placeholders = paths.map(() => '?').join(', ');\n    const selected = writableColumns.map(quoteIdent).join(', ');\n    const existing = await withNativeStatement(native, `SELECT ${selected} FROM frontmatter WHERE \"path\" IN (${placeholders})`, async (stmt) => {\n      stmt.bind(paths);\n      const reader = await stmt.runAndReadAll();\n      // Native wrappers distinguish numeric tags and SQL NULL from VARIANT null.\n      return new Map(reader.getRowObjects().map((row) => [row.path as string, row]));\n    });\n    const plans = chunk.map((row) => {\n      const old = existing.get(row[pathIndex] as string);\n      const changed = old ? writableColumns.flatMap((column, index) => (sameValue(old[column], row[index], column) ? [] : [index])) : [];\n      return { row, missing: old === undefined, changed, signature: changed.join(',') };\n    });\n    for (let position = 0; position < plans.length; ) {\n      const plan = plans[position];\n      if (!plan.missing && plan.changed.length === 0) {\n        ++position;\n        continue;\n      }\n      let end = position + 1;\n      while (end < plans.length && plans[end].missing === plan.missing && plans[end].signature === plan.signature) ++end;\n      if (plan.missing) {\n        await appendRows(\n          conn,\n          'frontmatter',\n          writableColumns,\n          insertSql,\n          plans.slice(position, end).map((repair) => repair.row)\n        );\n      } else {\n        const assignments = plan.changed.map((index) => `${quoteIdent(writableColumns[index])} = ?`).join(', ');\n        await withNativeStatement(native, `UPDATE frontmatter SET ${assignments} WHERE \"path\" = ?`, async (stmt) => {\n          for (let rowIndex = position; rowIndex < end; ++rowIndex) {\n            const row = plans[rowIndex].row;\n            stmt.bind([...plan.changed.map((index) => row[index] ?? null), row[pathIndex]] as DuckDBValue[]);\n            await stmt.run();\n          }\n        });\n      }\n      position = end;\n    }\n  }\n}\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], delta: ReconcileDelta): Promise<void> {\n  const added = new Set(delta.added);\n  const reparsed = docs.filter((doc) => !added.has(doc.relPath));\n  const changed = new Set(delta.added);\n  const removed: string[] = [];\n  for (let i = 0; i < reparsed.length; i += CONTENT_LOOKUP_CHUNK) {\n    const chunk = reparsed.slice(i, i + CONTENT_LOOKUP_CHUNK);\n    const stmt = await conn.prepare(`SELECT \"path\", title, summary, text FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n    const rows = (await stmt.all(...chunk.map((doc) => doc.relPath))) as ContentRow[];\n    const existing = new Map(rows.map((row) => [row.path, row]));\n    for (const doc of chunk) {\n      const row = existing.get(doc.relPath);\n      if (row && row.title === doc.search.title && row.summary === doc.search.summary && row.text === doc.search.text) continue;\n      changed.add(doc.relPath);\n      if (row) removed.push(doc.relPath);\n    }\n  }\n\n  const docPaths = new Set(docs.map((doc) => doc.relPath));\n  const vanished = touched.filter((path) => !docPaths.has(path));\n  for (let i = 0; i < vanished.length; i += CONTENT_LOOKUP_CHUNK) {\n    const chunk = vanished.slice(i, i + CONTENT_LOOKUP_CHUNK);\n    const stmt = await conn.prepare(`SELECT \"path\" FROM content WHERE \"path\" IN (${chunk.map(() => '?').join(', ')})`);\n    const rows = (await stmt.all(...chunk)) as Array<{ path: string }>;\n    removed.push(...rows.map((row) => row.path));\n  }\n\n  if (removed.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      removed.map((p) => [p])\n    );\n  // Keep document order and the existing insert/error semantics for added paths.\n  const changedDocs = docs.filter((doc) => changed.has(doc.relPath));\n  await appendRows(conn, 'content', CONTENT_COLUMNS, INSERT_CONTENT_SQL, changedDocs.map(contentRow));\n  // Commit the stale mark with actual content changes. Identical rows leave prior staleness\n  // untouched, and FTS preparation still owns missing-index recovery.\n  if (removed.length > 0 || changedDocs.length > 0) await markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  updateFrontmatter,\n  reconcileContent,\n};\n"],"names":["duckdbDialect","CONTENT_COLUMNS","INSERT_CONTENT_SQL","map","quoteIdent","join","CONTENT_LOOKUP_CHUNK","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","FRONTMATTER_UPDATE_CHUNK","withNativeStatement","native","sql","fn","stmt","errors","result","err","prepare","push","destroySync","length","AggregateError","updateFrontmatter","conn","writableColumns","rows","DuckDBVariantValue","DuckDBTypeId","physical","writable","pathIndex","columnTypes","insertSql","sameValue","i","chunk","paths","placeholders","selected","existing","plans","position","plan","end","assignments","missing","changed","signature","appendRows","slice","repair","row","index","rowIndex","bind","run","reader","runAndReadAll","Map","getRowObjects","path","old","get","flatMap","column","undefined","duckdb","duckdbApi","getRowObjectsJS","name","Error","type","value","Object","is","typeId","VARCHAR","HUGEINT","Number","isInteger","DOUBLE","reconcileContent","touched","docs","delta","added","reparsed","removed","docPaths","vanished","changedDocs","Set","filter","has","all","add","runBatch","p","markContentStale","addColumns","names","exec","beginMode","checkColumnLimit","count","SenseError"],"mappings":";;;;+BAgLaA;;;eAAAA;;;wBA/Kc;wBAGY;yBAGN;wBACP;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,qFAAqF;AACrF,kGAAkG;AAClG,iFAAiF;AACjF,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,IAAMC,kBAAkB;IAAC;IAAQ;IAAS;IAAW;CAAO;AAC5D,IAAMC,qBAAqB,AAAC,wBAAkE,OAA3CD,gBAAgBE,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM;AAC9F,wFAAwF;AACxF,IAAMC,uBAAuB;AAS7B,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,IAAMC,0BAA0B;AAChC,IAAMC,2BAA2B;AAEjC,SAAeC,oBAAuBC,MAAwB,EAAEC,GAAW,EAAEC,EAAiD;;YACtHC,MACAC,QACFC,QAGKC;;;;oBALI;;wBAAMN,OAAOO,OAAO,CAACN;;;oBAA5BE,OAAO;oBACPC;;;;;;;;;oBAGK;;wBAAMF,GAAGC;;;oBAAlBE,SAAS;;;;;;oBACFC;oBACPF,OAAOI,IAAI,CAACF;;;;;;oBAEd,IAAI;wBACFH,KAAKM,WAAW;oBAClB,EAAE,OAAOH,KAAK;wBACZF,OAAOI,IAAI,CAACF;oBACd;oBACA,IAAIF,OAAOM,MAAM,KAAK,GAAG,MAAMN,MAAM,CAAC,EAAE;oBACxC,IAAIA,OAAOM,MAAM,GAAG,GAAG,MAAM,IAAIC,eAAeP,QAAQ;oBACxD;;wBAAOC;;;;IACT;;AAEA,4EAA4E;AAC5E,gFAAgF;AAChF,SAAeO,kBAAkBC,IAAgB,EAAEC,eAAyB,EAAEC,IAAiB;;mBAEvFf,QACuC,MAArCgB,oBAAoBC,cACtBC,UAIAC,UACAC,WAEAC,aACAC,WACAC,WAOGC;;;;;mCACDC,OACAC,OACAC,cACAC,UACAC,UAMAC,OAKGC;;;;;4CACDC,MAKFC,KAWIC;;;;oDAhBFF,OAAOF,KAAK,CAACC,UAAS;oDAC5B,IAAI,CAACC,KAAKG,OAAO,IAAIH,KAAKI,OAAO,CAAC1B,MAAM,KAAK,GAAG;wDAC9C,EAAEqB;wDACF;;mFAAA;;oDACF;oDACIE,MAAMF,YAAW;oDACrB,MAAOE,MAAMH,MAAMpB,MAAM,IAAIoB,KAAK,CAACG,IAAI,CAACE,OAAO,KAAKH,KAAKG,OAAO,IAAIL,KAAK,CAACG,IAAI,CAACI,SAAS,KAAKL,KAAKK,SAAS,CAAE,EAAEJ;yDAC3GD,KAAKG,OAAO,EAAZH;;;;oDACF;;wDAAMM,IAAAA,oBAAU,EACdzB,MACA,eACAC,iBACAQ,WACAQ,MAAMS,KAAK,CAACR,WAAUE,KAAK/C,GAAG,CAAC,SAACsD;mEAAWA,OAAOC,GAAG;;;;oDALvD;;;;;;oDAQMP,cAAcF,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;+DAAU,AAAC,GAAqC,OAAnCvD,IAAAA,oBAAU,EAAC2B,eAAe,CAAC4B,MAAM,GAAE;uDAAOtD,IAAI,CAAC;oDAClG;;wDAAMW,oBAAoBC,QAAQ,AAAC,0BAAqC,OAAZkC,aAAY,sBAAoB,SAAO/B;;2EACxFwC;;;;;oFACDF;;;;4FAAAA,MAAMX,KAAK,CAACa,SAAS,CAACF,GAAG;4FAC/BtC,KAAKyC,IAAI,CAAC,AAAC,qBAAGZ,KAAKI,OAAO,CAAClD,GAAG,CAAC,SAACwD;oGAAUD;wGAAAA,aAAAA,GAAG,CAACC,MAAM,cAAVD,wBAAAA,aAAc;;gGAAOA,GAAG,CAACrB,UAAU;;4FAC7E;;gGAAMjB,KAAK0C,GAAG;;;4FAAd;;;;;;4EACF;4EAJSF,WAAWZ;;;iFAAUY,CAAAA,WAAWV,GAAE;;;;;;;;;;;;4EAAG,EAAEU;;;;;;;;;;;4DAKlD;;;;oDANA;;;oDAQFZ,YAAWE;;;;;;;oCACb;oCA1CMR,QAAQV,KAAKwB,KAAK,CAACf,GAAGA,IAAI1B;oCAC1B4B,QAAQD,MAAMvC,GAAG,CAAC,SAACuD;+CAAQA,GAAG,CAACrB,UAAU;;oCACzCO,eAAeD,MAAMxC,GAAG,CAAC;+CAAM;uCAAKE,IAAI,CAAC;oCACzCwC,WAAWd,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC;oCACrC;;wCAAMW,oBAAoBC,QAAQ,AAAC,UAAuD2B,OAA9CC,UAAS,uCAAkD,OAAbD,cAAa,MAAI,SAAOxB;;oDAE3H2C;;;;4DADN3C,KAAKyC,IAAI,CAAClB;4DACK;;gEAAMvB,KAAK4C,aAAa;;;4DAAjCD,SAAS;4DACf,2EAA2E;4DAC3E;;gEAAO,IAAIE,IAAIF,OAAOG,aAAa,GAAG/D,GAAG,CAAC,SAACuD;2EAAQ;wEAACA,IAAIS,IAAI;wEAAYT;qEAAI;;;;;4CAC9E;;;;oCALMZ,WAAW;oCAMXC,QAAQL,MAAMvC,GAAG,CAAC,SAACuD;wCACvB,IAAMU,MAAMtB,SAASuB,GAAG,CAACX,GAAG,CAACrB,UAAU;wCACvC,IAAMgB,UAAUe,MAAMrC,gBAAgBuC,OAAO,CAAC,SAACC,QAAQZ;mDAAWnB,UAAU4B,GAAG,CAACG,OAAO,EAAEb,GAAG,CAACC,MAAM,EAAEY,UAAU,EAAE,GAAG;gDAACZ;6CAAM;6CAAK,EAAE;wCAClI,OAAO;4CAAED,KAAAA;4CAAKN,SAASgB,QAAQI;4CAAWnB,SAAAA;4CAASC,WAAWD,QAAQhD,IAAI,CAAC;wCAAK;oCAClF;oCACS2C,WAAW;;;yCAAGA,CAAAA,WAAWD,MAAMpB,MAAM,AAAD;;;;;;;;;;;;;;;;;;;;;;oBA4B/C;oBA/DA,IAAIK,KAAKL,MAAM,KAAK,GAAG;;;oBACjBV,SAAS,AAACa,KAA0B2C,MAAM;oBACH;;wBAAMC,IAAAA,mBAAS;;;oBAAf,OAAA,eAArCzC,qBAAqC,KAArCA,oBAAoBC,eAAiB,KAAjBA;oBACX;;wBAAMlB,oBAAoBC,QAAQ,kCAAkC,SAAOG;;oCACpF2C;;;;4CAAS;;gDAAM3C,KAAK4C,aAAa;;;4CAAjCD,SAAS;4CACf;;gDAAOA,OAAOY,eAAe;;;;4BAC/B;;;;oBAHMxC,WAAW;oBAIXC,WAAW,IAAI6B,IAAIlC,gBAAgB5B,GAAG,CAAC,SAACyE,MAAMnC;+BAAM;4BAACmC;4BAAMnC;yBAAE;;oBAC7DJ,YAAYD,SAASiC,GAAG,CAAC;oBAC/B,IAAIhC,cAAcmC,WAAW,MAAM,IAAIK,MAAM;oBACvCvC,cAAc,IAAI2B,IAAI9B,SAAShC,GAAG,CAAC,SAACoE;+BAAW;4BAACA,OAAOK,IAAI;4BAAEL,OAAOO,IAAI;yBAAC;;oBACzEvC,YAAY,AAAC,4BAAkFR,OAAvDA,gBAAgB5B,GAAG,CAACC,oBAAU,EAAEC,IAAI,CAAC,OAAM,cAAsD,OAA1C0B,gBAAgB5B,GAAG,CAAC;+BAAM;uBAAKE,IAAI,CAAC,OAAM;oBACzImC,YAAY,mBAAC4B,KAAkBW,OAAgBR;4BAKpBH;wBAJ/B,IAAIW,UAAU,QAAQA,UAAUP,WAAW,OAAOJ,QAAQ;wBAC1D,IAAI9B,YAAY+B,GAAG,CAACE,YAAY,WAAW,OAAOS,OAAOC,EAAE,CAACb,KAAKW;wBACjE,IAAI,CAAEX,AAAG,YAAHA,KAAenC,qBAAqB,OAAO;wBACjD,IAAMiD,SAAS,OAAOH,UAAU,WAAW7C,aAAaiD,OAAO,GAAG,CAAA,OAAOJ,sCAAP,SAAOA,MAAI,MAAM,WAAW7C,aAAakD,OAAO,GAAG,OAAOL,UAAU,YAAY,CAACM,OAAOC,SAAS,CAACP,SAAS7C,aAAaqD,MAAM,GAAGf;wBACnM,OAAOU,WAAWV,aAAaJ,EAAAA,YAAAA,IAAIU,IAAI,cAARV,gCAAAA,UAAUc,MAAM,MAAKA,UAAUF,OAAOC,EAAE,CAACb,IAAIW,KAAK,EAAEA;oBACrF;oBACStC,IAAI;;;yBAAGA,CAAAA,IAAIT,KAAKL,MAAM,AAAD;;;;;;;;;;;;oBAAGc,KAAK1B;;;;;;;;;;;IA6CxC;;AAEA,SAAeyE,iBAAiB1D,IAAgB,EAAE2D,OAAiB,EAAEC,IAAiB,EAAEC,KAAqB;;YACrGC,OACAC,UACAxC,SACAyC,SACGrD,GAGarB,OAFdsB,OACAtB,MACAY,MACAc,UACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMtC,KACHkD,KAOJqC,UACAC,UACGvD,IAGarB,QACpB0E,UAHMpD,QACAtB,OACAY,OAUFiE;;;;oBAhCAL,QAAQ,IAAIM,IAAIP,MAAMC,KAAK;oBAC3BC,WAAWH,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ,CAACoF,MAAMQ,GAAG,CAAC5F,IAAIC,OAAO;;oBACtD4C,UAAU,IAAI6C,IAAIP,MAAMC,KAAK;oBAC7BE;oBACGrD,IAAI;;;yBAAGA,CAAAA,IAAIoD,SAASlE,MAAM,AAAD;;;;oBAC1Be,QAAQmD,SAASrC,KAAK,CAACf,GAAGA,IAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,qEAAoG,OAAhCkB,MAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAA/He,OAAO;oBACC;;wBAAMA,CAAAA,QAAAA,MAAKiF,GAAG,OAARjF,OAAS,qBAAGsB,MAAMvC,GAAG,CAAC,SAACK;mCAAQA,IAAIC,OAAO;;;;oBAAxDuB,OAAQ;oBACRc,WAAW,IAAImB,IAAIjC,KAAK7B,GAAG,CAAC,SAACuD;+BAAQ;4BAACA,IAAIS,IAAI;4BAAET;yBAAI;;oBACrD,kCAAA,2BAAA;;wBAAL,IAAK,YAAahB,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdlC,MAAN;4BACGkD,MAAMZ,SAASuB,GAAG,CAAC7D,IAAIC,OAAO;4BACpC,IAAIiD,OAAOA,IAAI/C,KAAK,KAAKH,IAAIE,MAAM,CAACC,KAAK,IAAI+C,IAAI9C,OAAO,KAAKJ,IAAIE,MAAM,CAACE,OAAO,IAAI8C,IAAI7C,IAAI,KAAKL,IAAIE,MAAM,CAACG,IAAI,EAAE;4BACjHwC,QAAQiD,GAAG,CAAC9F,IAAIC,OAAO;4BACvB,IAAIiD,KAAKoC,QAAQrE,IAAI,CAACjB,IAAIC,OAAO;wBACnC;;wBALK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAL8BgC,KAAKnC;;;;;;oBAapCyF,WAAW,IAAIG,IAAIR,KAAKvF,GAAG,CAAC,SAACK;+BAAQA,IAAIC,OAAO;;oBAChDuF,WAAWP,QAAQU,MAAM,CAAC,SAAChC;+BAAS,CAAC4B,SAASK,GAAG,CAACjC;;oBAC/C1B,KAAI;;;yBAAGA,CAAAA,KAAIuD,SAASrE,MAAM,AAAD;;;;oBAC1Be,SAAQsD,SAASxC,KAAK,CAACf,IAAGA,KAAInC;oBACvB;;wBAAMwB,KAAKN,OAAO,CAAC,AAAC,+CAA8E,OAAhCkB,OAAMvC,GAAG,CAAC;mCAAM;2BAAKE,IAAI,CAAC,OAAM;;;oBAAzGe,QAAO;oBACC;;wBAAMA,CAAAA,SAAAA,OAAKiF,GAAG,OAARjF,QAAS,qBAAGsB;;;oBAA1BV,QAAQ;oBACd8D,CAAAA,WAAAA,SAAQrE,IAAI,OAAZqE,UAAa,qBAAG9D,MAAK7B,GAAG,CAAC,SAACuD;+BAAQA,IAAIS,IAAI;;;;oBAJP1B,MAAKnC;;;;;;yBAOtCwF,CAAAA,QAAQnE,MAAM,GAAG,CAAA,GAAjBmE;;;;oBACF;;wBAAMhE,KAAKyE,QAAQ,CACjB,wCACAT,QAAQ3F,GAAG,CAAC,SAACqG;mCAAM;gCAACA;6BAAE;;;;oBAFxB;;;oBAIF,+EAA+E;oBACzEP,cAAcP,KAAKS,MAAM,CAAC,SAAC3F;+BAAQ6C,QAAQ+C,GAAG,CAAC5F,IAAIC,OAAO;;oBAChE;;wBAAM8C,IAAAA,oBAAU,EAACzB,MAAM,WAAW7B,iBAAiBC,oBAAoB+F,YAAY9F,GAAG,CAACI;;;oBAAvF;yBAGIuF,CAAAA,QAAQnE,MAAM,GAAG,KAAKsE,YAAYtE,MAAM,GAAG,CAAA,GAA3CmE;;;;oBAA8C;;wBAAMW,IAAAA,2BAAgB,EAAC3E;;;oBAAvB;;;;;;;;IACpD;;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,SAAe4E,WAAW5E,IAAgB,EAAE6E,KAAe;;;;;oBACzD,IAAIA,MAAMhF,MAAM,KAAK,GAAG;;;oBACxB;;wBAAMG,KAAK8E,IAAI,CAACD,MAAMxG,GAAG,CAAC,SAACyE;mCAAS,AAAC,sCAAsD,OAAjBxE,IAAAA,oBAAU,EAACwE,OAAM;2BAAWvE,IAAI,CAAC;;;oBAA3G;;;;;;IACF;;AAEO,IAAML,gBAAkC;IAC7C6G,WAAW,SAAXA;eAAiB;;IACjBC,kBAAAA,SAAAA,iBAAiBC,KAAK;QACpB,IAAIA,QAAQjG,yBAAyB;YACnC,MAAM,IAAIkG,oBAAU,CAAC,gBAAgB,AAAC,0BAA+ElG,OAAtDiG,OAAM,kDAAwE,OAAxBjG,yBAAwB;QAC/I;IACF;IACA4F,YAAAA;IACA7E,mBAAAA;IACA2D,kBAAAA;AACF"}

package/dist/cjs/store/duckdb/sql-functions.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).\n// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own\n  // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","duckdbApi","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","hasImpl","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","basenameImpl","undefined","textVec","segmentMatch","String"],"mappings":";;;;+BAOsBA;;;eAAAA;;;yBANO;8BACS;wBACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAInB,SAAeA,kBAAkBC,IAAsB;;YACW,MAA/DC,mBAAmBC,sBAAsBC;;;;oBAAsB;;wBAAMC,IAAAA,mBAAS;;;oBAAf,OAAA,eAA/DH,oBAA+D,KAA/DA,mBAAmBC,uBAA4C,KAA5CA,sBAAsBC,oBAAsB,KAAtBA;oBACjDH,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;4BAAEN,kBAAkBM,QAAQ;;wBACvEC,YAAYT,kBAAkBQ,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;4BAC1C,IAAMC,WAAWF,WAAWG,eAAe,CAAC;4BAC5C,IAAMC,WAAWJ,WAAWG,eAAe,CAAC;4BAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGG,IAAAA,uBAAO,EAACN,SAASO,OAAO,CAACJ,IAAID,SAASK,OAAO,CAACJ;4BACpHJ,aAAaS,KAAK;wBACpB;oBACF;oBAEFvB,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;;wBAC3Ce,aAAarB,kBAAkBM,QAAQ;wBACvCC,YAAYP,kBAAkBM,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;4BAC1C,IAAMW,UAAUZ,WAAWG,eAAe,CAAC;4BAC3C,IAAMU,YAAYb,WAAWc,WAAW,GAAG,IAAId,WAAWG,eAAe,CAAC,KAAK;4BAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGU,IAAAA,4BAAY,EAACH,QAAQH,OAAO,CAACJ,IAAIQ,YAAYA,UAAUJ,OAAO,CAACJ,KAAKW;4BAC1If,aAAaS,KAAK;wBACpB;oBACF;oBAEF,mIAAmI;oBACnI,uLAAuL;oBACvLvB,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;;wBAC3CC,YAAYP,kBAAkBM,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gCAEgDgB;4BAD1F,IAAMA,UAAUjB,WAAWG,eAAe,CAAC;4BAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGa,IAAAA,uBAAY,EAACC,QAAOF,mBAAAA,QAAQR,OAAO,CAACJ,gBAAhBY,8BAAAA,mBAAsB;4BAChHhB,aAAaS,KAAK;wBACpB;oBACF;;;;;;IAEJ"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/sql-functions.ts"],"sourcesContent":["import type { DuckDBConnection } from '@duckdb/node-api';\nimport { segmentMatch } from '../../text/segment.ts';\nimport { basenameImpl, hasImpl } from '../sql-functions.ts';\nimport { duckdbApi } from './native.ts';\n\n// Dynamic, not a top-level import: sqlite trees must never attempt to resolve this optional peer dependency (open.ts's connect()).\n// Goes through native.ts's shared duckdbApi(), not its own bare import, since a second independent lookup would hit Node's stale pre-install resolution miss.\nexport async function registerFunctions(conn: DuckDBConnection): Promise<void> {\n  const { DuckDBIntegerType, DuckDBScalarFunction, DuckDBVarCharType } = await duckdbApi();\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'has',\n      parameterTypes: [DuckDBVarCharType.instance, DuckDBVarCharType.instance],\n      returnType: DuckDBIntegerType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const fieldVec = inputChunk.getColumnVector(0);\n        const valueVec = inputChunk.getColumnVector(1);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, hasImpl(fieldVec.getItem(i), valueVec.getItem(i)));\n        outputVector.flush();\n      },\n    })\n  );\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'basename',\n      parameterTypes: [DuckDBVarCharType.instance],\n      varArgsType: DuckDBVarCharType.instance,\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const pathVec = inputChunk.getColumnVector(0);\n        const suffixVec = inputChunk.columnCount > 1 ? inputChunk.getColumnVector(1) : null;\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, basenameImpl(pathVec.getItem(i), suffixVec ? suffixVec.getItem(i) : undefined));\n        outputVector.flush();\n      },\n    })\n  );\n  // segment() rewrites unspaced-script runs into the grapheme phrase FTS5's `_seg` sidecars need (text/segment.ts); this store's own\n  // lexical index uses contains() instead and never calls this, but hand-written raw SQL naming it gets the real implementation, not a silent passthrough (PRINCIPLES: no-silent-modes).\n  conn.registerScalarFunction(\n    DuckDBScalarFunction.create({\n      name: 'segment',\n      parameterTypes: [DuckDBVarCharType.instance],\n      returnType: DuckDBVarCharType.instance,\n      mainFunction(_info, inputChunk, outputVector) {\n        const textVec = inputChunk.getColumnVector(0);\n        for (let i = 0; i < inputChunk.rowCount; i++) outputVector.setItem(i, segmentMatch(String(textVec.getItem(i) ?? '')));\n        outputVector.flush();\n      },\n    })\n  );\n}\n"],"names":["registerFunctions","conn","DuckDBIntegerType","DuckDBScalarFunction","DuckDBVarCharType","duckdbApi","registerScalarFunction","create","name","parameterTypes","instance","returnType","mainFunction","_info","inputChunk","outputVector","fieldVec","getColumnVector","valueVec","i","rowCount","setItem","hasImpl","getItem","flush","varArgsType","pathVec","suffixVec","columnCount","basenameImpl","undefined","textVec","segmentMatch","String"],"mappings":";;;;+BAOsBA;;;eAAAA;;;yBANO;8BACS;wBACZ;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAInB,SAAeA,kBAAkBC,IAAsB;;YACW,MAA/DC,mBAAmBC,sBAAsBC;;;;oBAAsB;;wBAAMC,IAAAA,mBAAS;;;oBAAf,OAAA,eAA/DH,oBAA+D,KAA/DA,mBAAmBC,uBAA4C,KAA5CA,sBAAsBC,oBAAsB,KAAtBA;oBACjDH,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;4BAAEN,kBAAkBM,QAAQ;;wBACvEC,YAAYT,kBAAkBQ,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;4BAC1C,IAAMC,WAAWF,WAAWG,eAAe,CAAC;4BAC5C,IAAMC,WAAWJ,WAAWG,eAAe,CAAC;4BAC5C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGG,IAAAA,uBAAO,EAACN,SAASO,OAAO,CAACJ,IAAID,SAASK,OAAO,CAACJ;4BACpHJ,aAAaS,KAAK;wBACpB;oBACF;oBAEFvB,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;;wBAC3Ce,aAAarB,kBAAkBM,QAAQ;wBACvCC,YAAYP,kBAAkBM,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;4BAC1C,IAAMW,UAAUZ,WAAWG,eAAe,CAAC;4BAC3C,IAAMU,YAAYb,WAAWc,WAAW,GAAG,IAAId,WAAWG,eAAe,CAAC,KAAK;4BAC/E,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGU,IAAAA,4BAAY,EAACH,QAAQH,OAAO,CAACJ,IAAIQ,YAAYA,UAAUJ,OAAO,CAACJ,KAAKW;4BAC1If,aAAaS,KAAK;wBACpB;oBACF;oBAEF,mIAAmI;oBACnI,uLAAuL;oBACvLvB,KAAKK,sBAAsB,CACzBH,qBAAqBI,MAAM,CAAC;wBAC1BC,MAAM;wBACNC,cAAc;4BAAGL,kBAAkBM,QAAQ;;wBAC3CC,YAAYP,kBAAkBM,QAAQ;wBACtCE,cAAAA,SAAAA,aAAaC,KAAK,EAAEC,UAAU,EAAEC,YAAY;gCAEgDgB;4BAD1F,IAAMA,UAAUjB,WAAWG,eAAe,CAAC;4BAC3C,IAAK,IAAIE,IAAI,GAAGA,IAAIL,WAAWM,QAAQ,EAAED,IAAKJ,aAAaM,OAAO,CAACF,GAAGa,IAAAA,uBAAY,EAACC,QAAOF,mBAAAA,QAAQR,OAAO,CAACJ,gBAAhBY,8BAAAA,mBAAsB;4BAChHhB,aAAaS,KAAK;wBACpB;oBACF;;;;;;IAEJ"}

package/dist/cjs/store/duckdb/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. segment() registers as a UDF (sql-functions.ts).\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;0BAC2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;CAAU;AAI5E,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,IAAMC,MAAMC,IAAAA,6BAAkB,EAACF;IAC/B,OAAO;QACLG,MAAM;QACNC,cAAcT;QACRU,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACf;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAgB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAChB,MAAMc,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOlB,IAAIkB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACtB;;YAC3BuB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,QAAQC,MAAM0B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,QAAQ6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAACrC,QAAQC,MAAM0B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAACtC,MAAMkC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGjC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBmC;;;;gCAAW;;oCAAM1C,OAAOQ,OAAO,CAACD;;;gCAAhCmC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE1C,MAAMsC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjF5D,OAAO6D,cAAc;oBACrB9D,SAAS+D,SAAS;;;;;YACpB;;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/store.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBInstance } from '@duckdb/node-api';\nimport { STORE_DIMS } from '../../embed/types.ts';\nimport { getColumns } from '../shared.ts';\nimport { withTransaction } from '../transaction.ts';\nimport type { Capability, Connection, Statement, Store } from '../types.ts';\nimport { hasVectorRow, pendingRows } from '../vectors.ts';\nimport { storeRowToJs } from './connection.ts';\nimport { fieldStats } from './fieldStats.ts';\nimport { createLexicalIndex } from './lexical.ts';\nimport { scanCandidates, scanSimilar, writeVectorBatch } from './vectors.ts';\n\n// Portable surface, links/sections/tags/rank, raw sql passthrough, lexical (fts BM25 + contains(), lexical.ts), and vectors (native\n// FLOAT[N] + array_cosine_similarity, vectors.ts) are implemented. segment() registers as a UDF (sql-functions.ts).\nexport const CAPABILITIES: ReadonlySet<Capability> = new Set(['vectors', 'segment']);\n\n// Shares one Connection instance (conn) with the builder's own reconcile call so transaction depth (transaction.ts) is tracked against the same object everywhere.\n// The instance is the native handle that owns the WAL: close() must close it, not just disconnect, or DuckDB never checkpoints and the next open reads a mismatched WAL.\nexport function createStore(instance: DuckDBInstance, duckdb: DuckDBConnection, conn: Connection): Store {\n  const lex = createLexicalIndex(conn);\n  return {\n    name: 'duckdb',\n    capabilities: CAPABILITIES,\n    async exec(sql: string): Promise<void> {\n      await conn.exec(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return conn.prepare(sql);\n    },\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      await conn.runBatch(sql, paramRows);\n    },\n    async transaction<T>(fn: () => Promise<T>): Promise<T> {\n      return withTransaction(conn, fn);\n    },\n    docs: {\n      async columns() {\n        return [...(await getColumns(conn))];\n      },\n      fieldStats: (columns, scopeWhere) => fieldStats(conn, columns, scopeWhere),\n    },\n    lexical: {\n      query: lex.query,\n    },\n    // The column's fixed DDL width (STORE_DIMS) is what every scan binds against, not the interface's per-call storeDims --\n    // see vectors.ts's padded() for why a shorter vector remains correct against a wider column.\n    vectors: {\n      pending: () => pendingRows(conn),\n      writeVectors: (rows) => writeVectorBatch(duckdb, conn, STORE_DIMS, rows),\n      candidates: (qv, _storeDims, fetch, allowed) => scanCandidates(duckdb, qv, STORE_DIMS, fetch, allowed),\n      similar: (path, opts) => scanSimilar(duckdb, conn, STORE_DIMS, path, opts),\n      hasVector: (path) => hasVectorRow(conn, path),\n    },\n    // No comparable setting to report yet.\n    async engineStatus() {\n      return {};\n    },\n    raw: {\n      async prepare(sql: string) {\n        const prepared = await duckdb.prepare(sql);\n        return {\n          columns: () => {\n            const out: Array<{ name: string }> = [];\n            for (let i = 0; i < prepared.columnCount; i++) out.push({ name: prepared.columnName(i) });\n            return out;\n          },\n          // Real incremental streaming: DuckDB reads its own result in row-group chunks, so\n          // each readUntil only fetches what isn't already in hand.\n          iterate: async function* (...params: unknown[]) {\n            if (params.length > 0) prepared.bind(params as Parameters<typeof prepared.bind>[0]);\n            const reader = await prepared.streamAndRead();\n            let yielded = 0;\n            do {\n              await reader.readUntil(reader.currentRowCount + 2048);\n              const rows = reader.getRowObjectsJS() as Array<Record<string, unknown>>;\n              for (; yielded < rows.length; yielded++) yield storeRowToJs(rows[yielded]);\n            } while (!reader.done);\n          },\n        };\n      },\n    },\n    async close() {\n      // Order matters: the connection must be gone before the instance closes the WAL.\n      duckdb.disconnectSync();\n      instance.closeSync();\n    },\n  };\n}\n"],"names":["CAPABILITIES","createStore","Set","instance","duckdb","conn","lex","createLexicalIndex","name","capabilities","exec","sql","prepare","runBatch","paramRows","transaction","fn","withTransaction","docs","columns","getColumns","fieldStats","scopeWhere","lexical","query","vectors","pending","pendingRows","writeVectors","rows","writeVectorBatch","STORE_DIMS","candidates","qv","_storeDims","fetch","allowed","scanCandidates","similar","path","opts","scanSimilar","hasVector","hasVectorRow","engineStatus","raw","prepared","out","i","columnCount","push","columnName","iterate","params","reader","yielded","length","bind","streamAndRead","readUntil","currentRowCount","getRowObjectsJS","storeRowToJs","done","close","disconnectSync","closeSync"],"mappings":";;;;;;;;;;;QAaaA;eAAAA;;QAIGC;eAAAA;;;uBAhBW;wBACA;6BACK;yBAEU;4BACb;4BACF;yBACQ;0BAC2B;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIvD,IAAMD,eAAwC,IAAIE,IAAI;IAAC;IAAW;CAAU;AAI5E,SAASD,YAAYE,QAAwB,EAAEC,MAAwB,EAAEC,IAAgB;IAC9F,IAAMC,MAAMC,IAAAA,6BAAkB,EAACF;IAC/B,OAAO;QACLG,MAAM;QACNC,cAAcT;QACRU,MAAN,SAAMA,KAAKC,GAAW;;;;;4BACpB;;gCAAMN,KAAKK,IAAI,CAACC;;;4BAAhB;;;;;;YACF;;QACMC,SAAN,SAAMA,QAAQD,GAAW;;;oBACvB;;wBAAON,KAAKO,OAAO,CAACD;;;YACtB;;QACME,UAAN,SAAMA,SAASF,GAAW,EAAEG,SAAsB;;;;;4BAChD;;gCAAMT,KAAKQ,QAAQ,CAACF,KAAKG;;;4BAAzB;;;;;;YACF;;QACMC,aAAN,SAAMA,YAAeC,EAAoB;;;oBACvC;;wBAAOC,IAAAA,8BAAe,EAACZ,MAAMW;;;YAC/B;;QACAE,MAAM;YACEC,SAAN,SAAMA;;;;;gCACQ;;oCAAMC,IAAAA,oBAAU,EAACf;;;gCAA7B;;oCAAQ;wCAAI;;;;;gBACd;;YACAgB,YAAY,SAAZA,WAAaF,SAASG;uBAAeD,IAAAA,wBAAU,EAAChB,MAAMc,SAASG;;QACjE;QACAC,SAAS;YACPC,OAAOlB,IAAIkB,KAAK;QAClB;QACA,wHAAwH;QACxH,6FAA6F;QAC7FC,SAAS;YACPC,SAAS,SAATA;uBAAeC,IAAAA,sBAAW,EAACtB;;YAC3BuB,cAAc,SAAdA,aAAeC;uBAASC,IAAAA,4BAAgB,EAAC1B,QAAQC,MAAM0B,mBAAU,EAAEF;;YACnEG,YAAY,SAAZA,WAAaC,IAAIC,YAAYC,OAAOC;uBAAYC,IAAAA,0BAAc,EAACjC,QAAQ6B,IAAIF,mBAAU,EAAEI,OAAOC;;YAC9FE,SAAS,SAATA,QAAUC,MAAMC;uBAASC,IAAAA,uBAAW,EAACrC,QAAQC,MAAM0B,mBAAU,EAAEQ,MAAMC;;YACrEE,WAAW,SAAXA,UAAYH;uBAASI,IAAAA,uBAAY,EAACtC,MAAMkC;;QAC1C;QAEMK,cADN,uCAAuC;QACvC,SAAMA;;;oBACJ;;wBAAO,CAAC;;;YACV;;QACAC,KAAK;YACGjC,SAAN,SAAMA,QAAQD,GAAW;;wBACjBmC;;;;gCAAW;;oCAAM1C,OAAOQ,OAAO,CAACD;;;gCAAhCmC,WAAW;gCACjB;;oCAAO;wCACL3B,SAAS,SAATA;4CACE,IAAM4B,MAA+B,EAAE;4CACvC,IAAK,IAAIC,IAAI,GAAGA,IAAIF,SAASG,WAAW,EAAED,IAAKD,IAAIG,IAAI,CAAC;gDAAE1C,MAAMsC,SAASK,UAAU,CAACH;4CAAG;4CACvF,OAAOD;wCACT;wCACA,kFAAkF;wCAClF,0DAA0D;wCAC1DK,SAAS,SAATA;4CAA0B,IAAA,IAAA,OAAA,UAAA,QAAA,AAAGC,SAAH,UAAA,OAAA,OAAA,GAAA,OAAA,MAAA;gDAAGA,OAAH,QAAA,SAAA,CAAA,KAAoB;;;oDAEtCC,QACFC,SAGI1B;;;;4DALR,IAAIwB,OAAOG,MAAM,GAAG,GAAGV,SAASW,IAAI,CAACJ;4DACtB;;gEAAA,uBAAMP,SAASY,aAAa;;;4DAArCJ,SAAS;4DACXC,UAAU;;;4DAEZ;;gEAAA,uBAAMD,OAAOK,SAAS,CAACL,OAAOM,eAAe,GAAG;;;4DAAhD;4DACM/B,OAAOyB,OAAOO,eAAe;;;iEAC5BN,CAAAA,UAAU1B,KAAK2B,MAAM,AAAD;;;;4DAAc;;gEAAMM,IAAAA,0BAAY,EAACjC,IAAI,CAAC0B,QAAQ;;;4DAAhC;;;4DAAXA;;;;;;gEACvB,CAACD,OAAOS,IAAI,EAAZ;;;;;;;;;;;4CACX;;oCACF;;;;gBACF;;QACF;QACMC,OAAN,SAAMA;;;oBACJ,iFAAiF;oBACjF5D,OAAO6D,cAAc;oBACrB9D,SAAS+D,SAAS;;;;;YACpB;;IACF;AACF"}

package/dist/cjs/store/duckdb/vectors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["createVectorWriteStage","scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","VECTOR_WRITE_STAGE","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","conn","rows","ARRAY","FLOAT","vectorType","duckdbApi","withTransaction","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","qv","fetch","allowed","arrayValue","DOUBLE","allowedList","query","queryNorm","score","sql","stmt","types","reader","size","sqrt","reduce","sum","value","prepare","map","bind","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","destroySync","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkBsBA;eAAAA;;QAqDAC;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BA7CU;yBAEO;wBACb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAChB,IAAMC,qBAAqB;AAGpB,SAAeN,uBAAuBO,MAAwB,EAAEC,IAAY;;;;;oBACjF;;wBAAMD,OAAOE,GAAG,CAAC,AAAC,qBAAoFD,OAAhEF,oBAAmB,+CAAkD,OAALE,MAAK;;;oBAA3G;;;;;;IACF;;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,IAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCzB;AAEJ;AAIO,SAAeL,iBAAiBI,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAE2B,IAAsB;;YAE5F,MAAjBC,OAAOC,OACTC;;;;oBAFN,IAAIH,KAAKjB,MAAM,KAAK,GAAG;;;oBACE;;wBAAMqB,IAAAA,mBAAS;;;oBAAf,OAAA,eAAjBH,QAAiB,KAAjBA,OAAOC,QAAU,KAAVA;oBACTC,aAAaF,MAAMC,OAAO7B;oBAEhC;;wBAAMgC,IAAAA,8BAAe,EAACN,MAAM;;oCAEpBO,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMhB;;;;4CAHb;;gDAAMlB,OAAOE,GAAG,CAAC,AAAC,eAAiC,OAAnBH;;;4CAAhC;4CACiB;;gDAAMC,OAAOmC,cAAc,CAACpC;;;4CAAvCmC,WAAW;4CACjB,IAAI;gDACG,kCAAA,2BAAA;;oDAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;wDAAbV,MAAN;wDACHgB,SAASE,aAAa,CAAClB,IAAImB,IAAI;wDAC/BH,SAASI,aAAa,CAACpB,IAAIqB,KAAK;wDAChCL,SAASM,WAAW,CAACvB,WAAWC,KAAKjB,OAAO8B;wDAC5CG,SAASO,MAAM;oDACjB;;oDALK;oDAAA;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;gDAMLP,SAASQ,SAAS;4CACpB,SAAU;gDACRR,SAASS,SAAS;4CACpB;4CACA;;gDAAM3C,OAAOE,GAAG,CAAC,AAAC,qDAAuE,OAAnBH,oBAAmB;;;4CAAzF;;;;;;4BACF;;;;oBAfA;;;;;;IAgBF;;AAIO,SAAeL,eAAeM,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BC,YAAYlB,OAAOmB,QACrBC,aACAC,OACAC,WACAC,OACAC,KAaAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBAzBR,IAAIkB,WAAWA,QAAQW,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMzB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrBC,cAAcH,UAAW,qBAAGA;oBAC5BI,QAAQ/C,OAAOyC,IAAI3C;oBACnBkD,YAAY1C,KAAKiD,IAAI,CAACR,MAAMS,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxET,QAAQD,cAAc,IAAI,QAAQ;oBAClCE,MAAM,AAAC,4QAOoBP,OAFmBM,OAAM,yEAEsC,OAA/DN,UAAU,AAAC,OAA6C,OAAvClC,SAAS,UAAUqC,YAAYtC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAI+C,cAAc;wBAAUJ,WAAWG;8BAAU,qBAAGD;wBAAaJ;;oBAC1FU,QAAsB,AAAC,qBAAIJ,cAAc;wBAAUtB,MAAMmB,QAAQ/C;8BAAS,qBAAGgD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACnHyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAE+B,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAE/GE,KAAKmB,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAe9E,YAAYK,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAEoC,IAAY,EAAEqC,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B9B,YAAYlB,OAAOmB,QACrB8B,aACA7B,aACAI,KAWAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBA1BR,IAAI8C,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACW,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM9B,KAAKmC,OAAO,CAAC;;;oBAAhCa,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAC1C;;;oBAAnCuC,aAAc;oBACpB,IAAIA,WAAWjE,MAAM,KAAK,GAAG;;;;oBACvBkE,UAAUG,IAAAA,uBAAY,EAACJ,YAAYb,GAAG,CAAC,SAAC7C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMW,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrB8B,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9BhC,cAAcyB,KAAK5B,OAAO,GAAI,qBAAG4B,KAAK5B,OAAO;oBAC7CO,MAAM,AAAC,+BAIPyB,OAJqCD,QAAQd,GAAG,CAAC;+BAAM;uBAAO/C,IAAI,CAAC,OAAM,kTAKzE0D,OADAI,YAAYnE,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYkE,YAAYnE,MAAM,KAAM,IAAG,YACd,OAAtE+D,KAAK5B,OAAO,GAAG,AAAC,OAA+C,OAAzClC,SAAS,YAAYqC,YAAYtC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAGyE,QAAQd,GAAG,CAAC,SAACmB;+BAAMnC,WAAW5C,OAAO+E,GAAGjF;;wBAASoC;uBAAM,qBAAGyC,cAAa,qBAAG7B;wBAAayB,KAAKS,CAAC;;oBACzH5B,QAAsB,AAAC,qBAAGsB,QAAQd,GAAG,CAAC;+BAAMlC,MAAMmB,QAAQ/C;;wBAAQJ;uBAAS,qBAAGiF,YAAYf,GAAG,CAAC;+BAAMlE;yBAAU,qBAAGoD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACvJyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAEkC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAEtEE,KAAKmB,WAAW;;;;;;;;;;IAEpB"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/duckdb/vectors.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBType, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, VectorCandidate, VectorSimilar, VectorWriteRow } from '../types.ts';\nimport { asCosine, sampleEvenly } from '../vectors.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store keeps vectors as native FLOAT[dims] arrays (dims fixed at DDL time by open.ts's ensureSchema, from embed/types.ts's STORE_DIMS) and\n// scans them with array_cosine_similarity, pushing top-k into SQL instead of pulling every row into JS; sqlite's int8+scale BLOB scan is its own representation. Every function here takes `dims` as a parameter rather than importing STORE_DIMS directly, so it stays testable at any width.\n\n// A value import of '@duckdb/node-api' must never sit at module top level, or a sqlite-only tree would resolve this optional dependency\n// just by importing store/index.ts; native.ts's shared duckdbApi() keeps that lazy and reuses whatever open.ts already resolved.\n\n// A bind position whose type is left to auto-inference (safe for plain strings/numbers; only\n// the vector ARRAY positions below need an explicit type -- see writeVectorBatch's comment).\nconst untyped = undefined as unknown as DuckDBType;\nconst VECTOR_WRITE_STAGE = '_sense_vector_write_stage';\n\n// Created once per native connection outside write transactions; its lifetime follows that connection.\nexport async function createVectorWriteStage(duckdb: DuckDBConnection, dims: number): Promise<void> {\n  await duckdb.run(`CREATE TEMP TABLE ${VECTOR_WRITE_STAGE} (\"path\" TEXT, chunk INTEGER, vector FLOAT[${dims}])`);\n}\n\n// The DDL-fixed array width can exceed a vector's actual length (a hypothetical model sliced under the column's width);\n// zero-padding leaves cosine scores unchanged since the added dimensions contribute nothing to either vector's dot product or norm.\nfunction padded(values: ArrayLike<number>, dims: number): number[] {\n  const out = new Array<number>(dims).fill(0);\n  for (let d = 0; d < Math.min(values.length, dims); d++) out[d] = values[d];\n  return out;\n}\n\nfunction inClause(column: string, count: number): string {\n  return `${column} IN (${Array.from({ length: count }, () => '?').join(', ')})`;\n}\n\n// Rows arrive int8-quantized with a per-vector scale (the wire format embed/query.ts produces for every store, see toStore);\n// dequantizing into a plain float array is this store's own representation choice, so both stores are handed the same vectors and diverge only here.\nfunction dequantize(row: VectorWriteRow, dims: number): number[] {\n  const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n  return padded(\n    Array.from(q, (v) => v * row.scale),\n    dims\n  );\n}\n\n// The native appender needs an explicit FLOAT array type: inference from an integer first component\n// silently truncates fractional components. The fixed UPDATE keeps each provider batch keyed by path and chunk.\nexport async function writeVectorBatch(duckdb: DuckDBConnection, conn: Connection, dims: number, rows: VectorWriteRow[]): Promise<void> {\n  if (rows.length === 0) return;\n  const { ARRAY, FLOAT } = await duckdbApi();\n  const vectorType = ARRAY(FLOAT, dims);\n\n  await withTransaction(conn, async () => {\n    await duckdb.run(`DELETE FROM ${VECTOR_WRITE_STAGE}`);\n    const appender = await duckdb.createAppender(VECTOR_WRITE_STAGE);\n    try {\n      for (const row of rows) {\n        appender.appendVarchar(row.path);\n        appender.appendInteger(row.chunk);\n        appender.appendArray(dequantize(row, dims), vectorType);\n        appender.endRow();\n      }\n      appender.flushSync();\n    } finally {\n      appender.closeSync();\n    }\n    await duckdb.run(`UPDATE embeddings SET vector = staged.vector FROM ${VECTOR_WRITE_STAGE} AS staged WHERE embeddings.\"path\" = staged.\"path\" AND embeddings.chunk = staged.chunk`);\n  });\n}\n\n// Best chunk per file by cosine, with the earliest chunk winning exact score ties.\n// An empty `allowed` set matches sqlite's scan (every row filtered out) without touching the table.\nexport async function scanCandidates(duckdb: DuckDBConnection, qv: Float32Array, dims: number, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  if (allowed && allowed.size === 0) return [];\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const allowedList = allowed ? [...allowed] : [];\n  const query = padded(qv, dims);\n  const queryNorm = Math.sqrt(query.reduce((sum, value) => sum + value * value, 0));\n  const score = queryNorm === 0 ? '0.0' : 'CASE WHEN array_inner_product(vector, vector) = 0 THEN 0.0 ELSE array_cosine_similarity(vector, ?) END';\n  const sql = `SELECT \"path\", start_line, end_line, score\n    FROM (\n      SELECT \"path\", start_line, end_line, score,\n        row_number() OVER (PARTITION BY \"path\" ORDER BY score DESC, chunk ASC) AS selected\n      FROM (\n        SELECT \"path\", chunk, start_line, end_line, ${score} AS score\n        FROM embeddings\n        WHERE vector IS NOT NULL ${allowed ? `AND ${inClause('\"path\"', allowedList.length)}` : ''}\n      ) sub\n    ) ranked\n    WHERE selected = 1\n    ORDER BY score DESC, \"path\" ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...(queryNorm === 0 ? [] : [arrayValue(query)]), ...allowedList, fetch];\n    const types: DuckDBType[] = [...(queryNorm === 0 ? [] : [ARRAY(DOUBLE, dims)]), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; start_line: number; end_line: number; score: number }>;\n    return rows.map((r) => ({ path: r.path, lines: `L${r.start_line}-${r.end_line}`, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, pushed into one native scan: the sampled target vectors become\n// a small VALUES list, cross-joined against every stored vector and reduced with max()/GROUP BY, so the O(target x stored) cost (12.7s at 201 chunks/note in JS) runs vectorized instead of scalar.\nexport async function scanSimilar(duckdb: DuckDBConnection, conn: Connection, dims: number, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  if (opts.allowed && opts.allowed.size === 0) return [];\n  const targetStmt = await conn.prepare('SELECT vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk');\n  const targetRows = (await targetStmt.all(path)) as Array<{ vector: number[] }>;\n  if (targetRows.length === 0) return [];\n  const targets = sampleEvenly(targetRows).map((row) => row.vector);\n\n  const { arrayValue, ARRAY, DOUBLE } = await duckdbApi();\n  const excludeList = [...opts.exclude];\n  const allowedList = opts.allowed ? [...opts.allowed] : [];\n  const sql = `WITH targets(tv) AS (VALUES ${targets.map(() => '(?)').join(', ')}), grouped AS (\n    SELECT e.\"path\" AS path, max(CASE WHEN array_inner_product(e.vector, e.vector) = 0 OR array_inner_product(t.tv, t.tv) = 0 THEN 0.0 ELSE array_cosine_similarity(e.vector, t.tv) END) AS score\n    FROM embeddings e, targets t\n    WHERE e.vector IS NOT NULL AND e.\"path\" != ?\n      ${excludeList.length > 0 ? `AND NOT ${inClause('e.\"path\"', excludeList.length)}` : ''}\n      ${opts.allowed ? `AND ${inClause('e.\"path\"', allowedList.length)}` : ''}\n    GROUP BY e.\"path\"\n  )\n    SELECT path, score FROM grouped\n    ORDER BY score DESC, path ASC\n    LIMIT ?`;\n  const stmt = await duckdb.prepare(sql);\n  try {\n    const values: DuckDBValue[] = [...targets.map((t) => arrayValue(padded(t, dims))), path, ...excludeList, ...allowedList, opts.k];\n    const types: DuckDBType[] = [...targets.map(() => ARRAY(DOUBLE, dims)), untyped, ...excludeList.map(() => untyped), ...allowedList.map(() => untyped), untyped];\n    stmt.bind(values, types);\n    const reader = await stmt.runAndReadAll();\n    const rows = reader.getRowObjectsJS() as Array<{ path: string; score: number }>;\n    return rows.map((r) => ({ path: r.path, similarity: asCosine(r.score) }));\n  } finally {\n    stmt.destroySync();\n  }\n}\n"],"names":["createVectorWriteStage","scanCandidates","scanSimilar","writeVectorBatch","untyped","undefined","VECTOR_WRITE_STAGE","duckdb","dims","run","padded","values","out","Array","fill","d","Math","min","length","inClause","column","count","from","join","dequantize","row","q","Int8Array","vector","buffer","byteOffset","byteLength","v","scale","conn","rows","ARRAY","FLOAT","vectorType","duckdbApi","withTransaction","appender","createAppender","appendVarchar","path","appendInteger","chunk","appendArray","endRow","flushSync","closeSync","qv","fetch","allowed","arrayValue","DOUBLE","allowedList","query","queryNorm","score","sql","stmt","types","reader","size","sqrt","reduce","sum","value","prepare","map","bind","runAndReadAll","getRowObjectsJS","r","lines","start_line","end_line","similarity","asCosine","destroySync","opts","targetStmt","targetRows","targets","excludeList","all","sampleEvenly","exclude","t","k"],"mappings":";;;;;;;;;;;QAkBsBA;eAAAA;;QAqDAC;eAAAA;;QAmCAC;eAAAA;;QA5DAC;eAAAA;;;6BA7CU;yBAEO;wBACb;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE1B,iJAAiJ;AACjJ,+RAA+R;AAE/R,wIAAwI;AACxI,iIAAiI;AAEjI,6FAA6F;AAC7F,6FAA6F;AAC7F,IAAMC,UAAUC;AAChB,IAAMC,qBAAqB;AAGpB,SAAeN,uBAAuBO,MAAwB,EAAEC,IAAY;;;;;oBACjF;;wBAAMD,OAAOE,GAAG,CAAC,AAAC,qBAAoFD,OAAhEF,oBAAmB,+CAAkD,OAALE,MAAK;;;oBAA3G;;;;;;IACF;;AAEA,wHAAwH;AACxH,oIAAoI;AACpI,SAASE,OAAOC,MAAyB,EAAEH,IAAY;IACrD,IAAMI,MAAM,IAAIC,MAAcL,MAAMM,IAAI,CAAC;IACzC,IAAK,IAAIC,IAAI,GAAGA,IAAIC,KAAKC,GAAG,CAACN,OAAOO,MAAM,EAAEV,OAAOO,IAAKH,GAAG,CAACG,EAAE,GAAGJ,MAAM,CAACI,EAAE;IAC1E,OAAOH;AACT;AAEA,SAASO,SAASC,MAAc,EAAEC,KAAa;IAC7C,OAAO,AAAC,GAAgBR,OAAdO,QAAO,SAA2D,OAApDP,MAAMS,IAAI,CAAC;QAAEJ,QAAQG;IAAM,GAAG;eAAM;OAAKE,IAAI,CAAC,OAAM;AAC9E;AAEA,6HAA6H;AAC7H,qJAAqJ;AACrJ,SAASC,WAAWC,GAAmB,EAAEjB,IAAY;IACnD,IAAMkB,IAAI,IAAIC,UAAUF,IAAIG,MAAM,CAACC,MAAM,EAAEJ,IAAIG,MAAM,CAACE,UAAU,EAAEL,IAAIG,MAAM,CAACG,UAAU;IACvF,OAAOrB,OACLG,MAAMS,IAAI,CAACI,GAAG,SAACM;eAAMA,IAAIP,IAAIQ,KAAK;QAClCzB;AAEJ;AAIO,SAAeL,iBAAiBI,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAE2B,IAAsB;;YAE5F,MAAjBC,OAAOC,OACTC;;;;oBAFN,IAAIH,KAAKjB,MAAM,KAAK,GAAG;;;oBACE;;wBAAMqB,IAAAA,mBAAS;;;oBAAf,OAAA,eAAjBH,QAAiB,KAAjBA,OAAOC,QAAU,KAAVA;oBACTC,aAAaF,MAAMC,OAAO7B;oBAEhC;;wBAAMgC,IAAAA,8BAAe,EAACN,MAAM;;oCAEpBO,UAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMhB;;;;4CAHb;;gDAAMlB,OAAOE,GAAG,CAAC,AAAC,eAAiC,OAAnBH;;;4CAAhC;4CACiB;;gDAAMC,OAAOmC,cAAc,CAACpC;;;4CAAvCmC,WAAW;4CACjB,IAAI;gDACG,kCAAA,2BAAA;;oDAAL,IAAK,YAAaN,2BAAb,6BAAA,QAAA,yBAAA,iCAAmB;wDAAbV,MAAN;wDACHgB,SAASE,aAAa,CAAClB,IAAImB,IAAI;wDAC/BH,SAASI,aAAa,CAACpB,IAAIqB,KAAK;wDAChCL,SAASM,WAAW,CAACvB,WAAWC,KAAKjB,OAAO8B;wDAC5CG,SAASO,MAAM;oDACjB;;oDALK;oDAAA;;;6DAAA,6BAAA;4DAAA;;;4DAAA;kEAAA;;;;gDAMLP,SAASQ,SAAS;4CACpB,SAAU;gDACRR,SAASS,SAAS;4CACpB;4CACA;;gDAAM3C,OAAOE,GAAG,CAAC,AAAC,qDAAuE,OAAnBH,oBAAmB;;;4CAAzF;;;;;;4BACF;;;;oBAfA;;;;;;IAgBF;;AAIO,SAAeL,eAAeM,MAAwB,EAAE4C,EAAgB,EAAE3C,IAAY,EAAE4C,KAAa,EAAEC,OAAqB;;YAE3F,MAA9BC,YAAYlB,OAAOmB,QACrBC,aACAC,OACAC,WACAC,OACAC,KAaAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBAzBR,IAAIkB,WAAWA,QAAQW,IAAI,KAAK,GAAG;;;;oBACG;;wBAAMzB,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrBC,cAAcH,UAAW,qBAAGA;oBAC5BI,QAAQ/C,OAAOyC,IAAI3C;oBACnBkD,YAAY1C,KAAKiD,IAAI,CAACR,MAAMS,MAAM,CAAC,SAACC,KAAKC;+BAAUD,MAAMC,QAAQA;uBAAO;oBACxET,QAAQD,cAAc,IAAI,QAAQ;oBAClCE,MAAM,AAAC,4QAOoBP,OAFmBM,OAAM,yEAEsC,OAA/DN,UAAU,AAAC,OAA6C,OAAvClC,SAAS,UAAUqC,YAAYtC,MAAM,KAAM,IAAG;oBAMnF;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAI+C,cAAc;wBAAUJ,WAAWG;8BAAU,qBAAGD;wBAAaJ;;oBAC1FU,QAAsB,AAAC,qBAAIJ,cAAc;wBAAUtB,MAAMmB,QAAQ/C;8BAAS,qBAAGgD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACnHyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAE+B,OAAO,AAAC,IAAmBD,OAAhBA,EAAEE,UAAU,EAAC,KAAc,OAAXF,EAAEG,QAAQ;gCAAIC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAE/GE,KAAKmB,WAAW;;;;;;;;;;IAEpB;;AAIO,SAAe9E,YAAYK,MAAwB,EAAE2B,IAAgB,EAAE1B,IAAY,EAAEoC,IAAY,EAAEqC,IAAgE;;YAElKC,YACAC,YAEAC,SAEgC,MAA9B9B,YAAYlB,OAAOmB,QACrB8B,aACA7B,aACAI,KAWAC,MAEElD,QACAmD,OAEAC,QACA5B;;;;oBA1BR,IAAI8C,KAAK5B,OAAO,IAAI4B,KAAK5B,OAAO,CAACW,IAAI,KAAK,GAAG;;;;oBAC1B;;wBAAM9B,KAAKmC,OAAO,CAAC;;;oBAAhCa,aAAa;oBACC;;wBAAMA,WAAWI,GAAG,CAAC1C;;;oBAAnCuC,aAAc;oBACpB,IAAIA,WAAWjE,MAAM,KAAK,GAAG;;;;oBACvBkE,UAAUG,IAAAA,uBAAY,EAACJ,YAAYb,GAAG,CAAC,SAAC7C;+BAAQA,IAAIG,MAAM;;oBAE1B;;wBAAMW,IAAAA,mBAAS;;;oBAAf,OAAA,eAA9Be,aAA8B,KAA9BA,YAAYlB,QAAkB,KAAlBA,OAAOmB,SAAW,KAAXA;oBACrB8B,cAAe,qBAAGJ,KAAKO,OAAO;oBAC9BhC,cAAcyB,KAAK5B,OAAO,GAAI,qBAAG4B,KAAK5B,OAAO;oBAC7CO,MAAM,AAAC,+BAIPyB,OAJqCD,QAAQd,GAAG,CAAC;+BAAM;uBAAO/C,IAAI,CAAC,OAAM,kTAKzE0D,OADAI,YAAYnE,MAAM,GAAG,IAAI,AAAC,WAAmD,OAAzCC,SAAS,YAAYkE,YAAYnE,MAAM,KAAM,IAAG,YACd,OAAtE+D,KAAK5B,OAAO,GAAG,AAAC,OAA+C,OAAzClC,SAAS,YAAYqC,YAAYtC,MAAM,KAAM,IAAG;oBAM/D;;wBAAMX,OAAO8D,OAAO,CAACT;;;oBAA5BC,OAAO;;;;;;;;;oBAELlD,SAAwB,AAAC,qBAAGyE,QAAQd,GAAG,CAAC,SAACmB;+BAAMnC,WAAW5C,OAAO+E,GAAGjF;;wBAASoC;uBAAM,qBAAGyC,cAAa,qBAAG7B;wBAAayB,KAAKS,CAAC;;oBACzH5B,QAAsB,AAAC,qBAAGsB,QAAQd,GAAG,CAAC;+BAAMlC,MAAMmB,QAAQ/C;;wBAAQJ;uBAAS,qBAAGiF,YAAYf,GAAG,CAAC;+BAAMlE;yBAAU,qBAAGoD,YAAYc,GAAG,CAAC;+BAAMlE;;wBAAUA;;oBACvJyD,KAAKU,IAAI,CAAC5D,QAAQmD;oBACH;;wBAAMD,KAAKW,aAAa;;;oBAAjCT,SAAS;oBACT5B,OAAO4B,OAAOU,eAAe;oBACnC;;wBAAOtC,KAAKmC,GAAG,CAAC,SAACI;mCAAO;gCAAE9B,MAAM8B,EAAE9B,IAAI;gCAAEkC,YAAYC,IAAAA,mBAAQ,EAACL,EAAEf,KAAK;4BAAE;;;;oBAEtEE,KAAKmB,WAAW;;;;;;;;;;IAEpB"}

package/dist/cjs/store/embed-scope.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n  provider: string;\n  model: string;\n  endpoint?: string;\n  chunkVersion: string;\n}\n\n// One embed segment's provider/model/endpoint/chunkVersion, ignoring any trailing `@identity`.\n// The endpoint-less HTTP shape remains readable so upgrading invalidates vectors, not core data.\nfunction parseEmbedSegment(sig: string): EmbedSegment | null {\n  const part = sig.split('|').find((p) => p.startsWith('embed:'));\n  if (part === undefined || part === 'embed:off') return null;\n  const at = part.indexOf('@');\n  const body = at === -1 ? part : part.slice(0, at);\n  const withEndpoint = /^embed:(openai|cohere):(.+):endpoint=([0-9a-f]{64}):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (withEndpoint) {\n    const [, provider, model, endpoint, chunkVersion] = withEndpoint;\n    return { provider, model, endpoint, chunkVersion };\n  }\n  const legacy = /^embed:(static|openai|cohere):(.+):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (!legacy) return null;\n  const [, provider, model, chunkVersion] = legacy;\n  return { provider, model, chunkVersion };\n}\n\n// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);\n// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.\nexport function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null {\n  const b = parseEmbedSegment(before);\n  const a = parseEmbedSegment(after);\n  if (b === null || a === null) return null;\n  if (b.chunkVersion !== a.chunkVersion) return 'chunk';\n  if (b.provider !== a.provider || b.model !== a.model || b.endpoint !== a.endpoint) return 'model';\n  return null;\n}\n"],"names":["classifyEmbedChange","parseEmbedSegment","sig","part","split","find","p","startsWith","undefined","at","indexOf","body","slice","withEndpoint","exec","provider","model","endpoint","chunkVersion","legacy","before","after","b","a"],"mappings":"AAAA,+FAA+F;AAC/F,oFAAoF;;;;;+BA+BpEA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AApBhB,+FAA+F;AAC/F,iGAAiG;AACjG,SAASC,kBAAkBC,GAAW;IACpC,IAAMC,OAAOD,IAAIE,KAAK,CAAC,KAAKC,IAAI,CAAC,SAACC;eAAMA,EAAEC,UAAU,CAAC;;IACrD,IAAIJ,SAASK,aAAaL,SAAS,aAAa,OAAO;IACvD,IAAMM,KAAKN,KAAKO,OAAO,CAAC;IACxB,IAAMC,OAAOF,OAAO,CAAC,IAAIN,OAAOA,KAAKS,KAAK,CAAC,GAAGH;IAC9C,IAAMI,eAAe,6EAA6EC,IAAI,CAACH;IACvG,IAAIE,cAAc;QAChB,IAAoDA,iCAAAA,kBAA3CE,WAA2CF,kBAAjCG,QAAiCH,kBAA1BI,WAA0BJ,kBAAhBK,eAAgBL;QACpD,OAAO;YAAEE,UAAAA;YAAUC,OAAAA;YAAOC,UAAAA;YAAUC,cAAAA;QAAa;IACnD;IACA,IAAMC,SAAS,4DAA4DL,IAAI,CAACH;IAChF,IAAI,CAACQ,QAAQ,OAAO;IACpB,IAA0CA,2BAAAA,YAAjCJ,YAAiCI,YAAvBH,SAAuBG,YAAhBD,gBAAgBC;IAC1C,OAAO;QAAEJ,UAAAA;QAAUC,OAAAA;QAAOE,cAAAA;IAAa;AACzC;AAIO,SAASlB,oBAAoBoB,MAAc,EAAEC,KAAa;IAC/D,IAAMC,IAAIrB,kBAAkBmB;IAC5B,IAAMG,IAAItB,kBAAkBoB;IAC5B,IAAIC,MAAM,QAAQC,MAAM,MAAM,OAAO;IACrC,IAAID,EAAEJ,YAAY,KAAKK,EAAEL,YAAY,EAAE,OAAO;IAC9C,IAAII,EAAEP,QAAQ,KAAKQ,EAAER,QAAQ,IAAIO,EAAEN,KAAK,KAAKO,EAAEP,KAAK,IAAIM,EAAEL,QAAQ,KAAKM,EAAEN,QAAQ,EAAE,OAAO;IAC1F,OAAO;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/open-source/ai/sensemaking/src/store/embed-scope.ts"],"sourcesContent":["// Decomposition of the `embed:` featureSignature segment (config/access.ts's featureSignature,\n// features/embed.ts's signature()), for open()'s narrow embed-invalidation routing.\n\nexport type EmbedChangeKind = 'model' | 'chunk';\n\ninterface EmbedSegment {\n  provider: string;\n  model: string;\n  endpoint?: string;\n  chunkVersion: string;\n}\n\n// One embed segment's provider/model/endpoint/chunkVersion, ignoring any trailing `@identity`.\n// The endpoint-less HTTP shape remains readable so upgrading invalidates vectors, not core data.\nfunction parseEmbedSegment(sig: string): EmbedSegment | null {\n  const part = sig.split('|').find((p) => p.startsWith('embed:'));\n  if (part === undefined || part === 'embed:off') return null;\n  const at = part.indexOf('@');\n  const body = at === -1 ? part : part.slice(0, at);\n  const withEndpoint = /^embed:(openai|cohere):(.+):endpoint=([0-9a-f]{64}):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (withEndpoint) {\n    const [, provider, model, endpoint, chunkVersion] = withEndpoint;\n    return { provider, model, endpoint, chunkVersion };\n  }\n  const legacy = /^embed:(static|openai|cohere):(.+):(chunk:v\\d+(?::\\d+)?)$/.exec(body);\n  if (!legacy) return null;\n  const [, provider, model, chunkVersion] = legacy;\n  return { provider, model, chunkVersion };\n}\n\n// 'chunk' when chunkTokens or the chunk version moved (boundaries changed, embeddings rebuild);\n// 'model' when only the provider or model moved (rows unchanged, values stale); null otherwise.\nexport function classifyEmbedChange(before: string, after: string): EmbedChangeKind | null {\n  const b = parseEmbedSegment(before);\n  const a = parseEmbedSegment(after);\n  if (b === null || a === null) return null;\n  if (b.chunkVersion !== a.chunkVersion) return 'chunk';\n  if (b.provider !== a.provider || b.model !== a.model || b.endpoint !== a.endpoint) return 'model';\n  return null;\n}\n"],"names":["classifyEmbedChange","parseEmbedSegment","sig","part","split","find","p","startsWith","undefined","at","indexOf","body","slice","withEndpoint","exec","provider","model","endpoint","chunkVersion","legacy","before","after","b","a"],"mappings":"AAAA,+FAA+F;AAC/F,oFAAoF;;;;;+BA+BpEA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AApBhB,+FAA+F;AAC/F,iGAAiG;AACjG,SAASC,kBAAkBC,GAAW;IACpC,IAAMC,OAAOD,IAAIE,KAAK,CAAC,KAAKC,IAAI,CAAC,SAACC;eAAMA,EAAEC,UAAU,CAAC;;IACrD,IAAIJ,SAASK,aAAaL,SAAS,aAAa,OAAO;IACvD,IAAMM,KAAKN,KAAKO,OAAO,CAAC;IACxB,IAAMC,OAAOF,OAAO,CAAC,IAAIN,OAAOA,KAAKS,KAAK,CAAC,GAAGH;IAC9C,IAAMI,eAAe,6EAA6EC,IAAI,CAACH;IACvG,IAAIE,cAAc;QAChB,IAAoDA,iCAAAA,kBAA3CE,WAA2CF,kBAAjCG,QAAiCH,kBAA1BI,WAA0BJ,kBAAhBK,eAAgBL;QACpD,OAAO;YAAEE,UAAAA;YAAUC,OAAAA;YAAOC,UAAAA;YAAUC,cAAAA;QAAa;IACnD;IACA,IAAMC,SAAS,4DAA4DL,IAAI,CAACH;IAChF,IAAI,CAACQ,QAAQ,OAAO;IACpB,IAA0CA,2BAAAA,YAAjCJ,YAAiCI,YAAvBH,SAAuBG,YAAhBD,gBAAgBC;IAC1C,OAAO;QAAEJ,UAAAA;QAAUC,OAAAA;QAAOE,cAAAA;IAAa;AACzC;AAIO,SAASlB,oBAAoBoB,MAAc,EAAEC,KAAa;IAC/D,IAAMC,IAAIrB,kBAAkBmB;IAC5B,IAAMG,IAAItB,kBAAkBoB;IAC5B,IAAIC,MAAM,QAAQC,MAAM,MAAM,OAAO;IACrC,IAAID,EAAEJ,YAAY,KAAKK,EAAEL,YAAY,EAAE,OAAO;IAC9C,IAAII,EAAEP,QAAQ,KAAKQ,EAAER,QAAQ,IAAIO,EAAEN,KAAK,KAAKO,EAAEP,KAAK,IAAIM,EAAEL,QAAQ,KAAKM,EAAEN,QAAQ,EAAE,OAAO;IAC1F,OAAO;AACT"}

package/dist/cjs/store/engine.d.cts

@@ -0,0 +1,28 @@
+import { SenseError } from '../errors.js';
+export declare const ENGINE_IDENTITY_META_KEY = "engine_identity";
+export declare function engineIdentity(pkg: string, version: string | undefined): string;
+/**
+ * One version-specific replacement of a store's SQL builders. `range` matches the engine version (the npm package's,
+ * or Node's bundled SQLite for sqlite), prereleases included since every `@duckdb/node-api` release is one;
+ * `reason` names the observed difference.
+ */
+export interface EnginePatch<Builders> {
+    range: string;
+    reason: string;
+    builders: Partial<Builders>;
+}
+export declare function selectBuilders<Builders extends object>(defaults: Builders, patches: readonly EnginePatch<Builders>[], version: string | undefined): Builders;
+export interface EngineCheckDoc {
+    path: string;
+    title: string;
+    summary: string;
+    text: string;
+}
+export declare const ENGINE_CHECK_DOCS: readonly EngineCheckDoc[];
+export declare const ENGINE_CHECK_QUERIES: ReadonlyArray<{
+    terms: string;
+    paths: string[];
+}>;
+export declare function npmRemedy(pkg: string, range: string): string;
+export declare function checkEngine(store: string, identity: string, remedy: string, query: (terms: string) => Promise<string[]>): Promise<void>;
+export declare function engineCheckFailed(store: string, identity: string, remedy: string, failure: string): SenseError;

package/dist/cjs/store/engine.d.ts

@@ -0,0 +1,28 @@
+import { SenseError } from '../errors.js';
+export declare const ENGINE_IDENTITY_META_KEY = "engine_identity";
+export declare function engineIdentity(pkg: string, version: string | undefined): string;
+/**
+ * One version-specific replacement of a store's SQL builders. `range` matches the engine version (the npm package's,
+ * or Node's bundled SQLite for sqlite), prereleases included since every `@duckdb/node-api` release is one;
+ * `reason` names the observed difference.
+ */
+export interface EnginePatch<Builders> {
+    range: string;
+    reason: string;
+    builders: Partial<Builders>;
+}
+export declare function selectBuilders<Builders extends object>(defaults: Builders, patches: readonly EnginePatch<Builders>[], version: string | undefined): Builders;
+export interface EngineCheckDoc {
+    path: string;
+    title: string;
+    summary: string;
+    text: string;
+}
+export declare const ENGINE_CHECK_DOCS: readonly EngineCheckDoc[];
+export declare const ENGINE_CHECK_QUERIES: ReadonlyArray<{
+    terms: string;
+    paths: string[];
+}>;
+export declare function npmRemedy(pkg: string, range: string): string;
+export declare function checkEngine(store: string, identity: string, remedy: string, query: (terms: string) => Promise<string[]>): Promise<void>;
+export declare function engineCheckFailed(store: string, identity: string, remedy: string, failure: string): SenseError;