sensemaking 0.21.0 → 0.22.0

package/dist/esm/config/validate.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport { type Config, STORE_NAMES, type StoreName } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'store', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\n// Exported so a docs test can assert every key here has a schema.json property -- one source\n// of truth for what the embed block accepts, checked against the other for drift.\nexport const KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages', 'chunkTokens']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\nfunction validateStoreKey(value: unknown, configPath: string): void {\n  if (!STORE_NAMES.includes(value as StoreName)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: store must be ${STORE_NAMES.map((name) => `\"${name}\"`).join(' or ')}`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n  if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that doesn't\n// need the top-level embed block: links is seeded by word-match rows, so declaring it without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string is rejected rather than inferred to be SQL: an entry says which of the two\n  // verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  // `content` (content.tokenize) is rejected by name for the same reason: every outcome it\n  // selected (unspaced-script search, stemming, quoted-phrase substring matching) is now\n  // automatic on every store, so a config that still sets it would silently get nothing back.\n  if (cfg.content !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize was removed -- every outcome it selected is now automatic on every store; remove the \"content\" block`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.store !== undefined) {\n    validateStoreKey(cfg.store, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","STORE_NAMES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateStoreKey","includes","map","name","join","validateFeaturesBlock","block","unknown","v","entries","validateEmbedBlock","embed","model","undefined","provider","url","key","languages","some","l","chunkTokens","Number","isInteger","isSignalName","isSignalWeight","isFinite","validatePresetSignals","signal","weight","names","requires","validatePreset","preset","exclude","signals","where","validateSavedQuery","JSON","stringify","slice","entry","sql","trim","search","validateConfig","checks","content","presets","default","queries","features","store"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AACnF,SAAsBC,WAAW,QAAwB,aAAa;AAEtE,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAS;CAAU;AACrG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,6FAA6F;AAC7F,kFAAkF;AAClF,OAAO,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;IAAa;CAAc,EAAE;AACzG,MAAMI,oBAAoB,IAAIJ,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASK,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACX,WAAWY,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,SAASG,iBAAiBX,KAAc,EAAEQ,UAAkB;IAC1D,IAAI,CAACvB,YAAY2B,QAAQ,CAACZ,QAAqB;QAC7C,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,gBAAgB,EAAEvB,YAAY4B,GAAG,CAAC,CAACC,OAAS,CAAC,CAAC,EAAEA,KAAK,CAAC,CAAC,EAAEC,IAAI,CAAC,SAAS;IAC9H;AACF;AAEA,SAASC,sBAAsBhB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMS,QAAQjB;IACd,MAAMkB,UAAUxB,OAAOC,IAAI,CAACsB,OAAOrB,MAAM,CAAC,CAACC,IAAM,CAACR,mBAAmBS,GAAG,CAACD;IACzE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEU,QAAQH,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACD,MAAMK,EAAE,IAAIzB,OAAO0B,OAAO,CAACH,OAAQ;QAC7C,IAAI,OAAOE,MAAM,WAAW;YAC1B,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEM,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASO,mBAAmBrB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMc,QAAQtB;IACd,MAAMkB,UAAUxB,OAAOC,IAAI,CAAC2B,OAAO1B,MAAM,CAAC,CAACC,IAAM,CAACP,iBAAiBQ,GAAG,CAACD;IACvE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEU,QAAQH,IAAI,CAAC,OAAO;IACxG;IACA,IAAIO,MAAMC,KAAK,KAAKC,aAAa,OAAOF,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAIzC,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIc,MAAMC,KAAK,KAAKC,aAAaF,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAIzC,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIc,MAAMG,QAAQ,KAAKD,aAAaF,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIc,MAAMI,GAAG,KAAKF,aAAa,OAAOF,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIc,MAAMK,GAAG,KAAKH,aAAa,OAAOF,MAAMK,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIc,MAAMG,QAAQ,KAAK,YAAaH,CAAAA,MAAMI,GAAG,KAAKF,aAAaF,MAAMI,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIc,MAAMM,SAAS,KAAKJ,aAAc,CAAA,CAACvB,MAAMC,OAAO,CAACoB,MAAMM,SAAS,KAAKN,MAAMM,SAAS,CAACzB,MAAM,KAAK,KAAKmB,MAAMM,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;IACA,IAAIc,MAAMS,WAAW,KAAKP,aAAc,CAAA,OAAOF,MAAMS,WAAW,KAAK,YAAY,CAACC,OAAOC,SAAS,CAACX,MAAMS,WAAW,KAAKT,MAAMS,WAAW,IAAI,CAAA,GAAI;QAChJ,MAAM,IAAIjD,WAAW,kBAAkB,GAAG0B,WAAW,8CAA8C,CAAC;IACtG;AACF;AAEA,SAAS0B,aAAalC,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmC6B,QAAQ,CAACZ;AACnF;AAEA,SAASmC,eAAenC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYgC,OAAOI,QAAQ,CAACpC,UAAUA,QAAQ;AACxE;AAEA,iGAAiG;AACjG,qHAAqH;AACrH,SAASqC,sBAAsBvB,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEM,KAAK,2FAA2F,EAAE/B,aAAagC,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMK,UAAU1B,OAAO0B,OAAO,CAACpB;IAC/B,MAAMkB,UAAUE,QAAQxB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACqC,aAAarC,IAAIgB,GAAG,CAAC,CAAC,CAAChB,EAAE,GAAKA;IACvE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,iCAAiC,EAAEI,QAAQH,IAAI,CAAC,MAAM,oBAAoB,EAAEhC,aAAagC,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAACuB,QAAQC,OAAO,IAAInB,QAAS;QACtC,IAAI,CAACe,eAAeI,SAAS;YAC3B,MAAM,IAAIzD,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,SAAS,EAAEwB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAIrD,IAAIiC,QAAQP,GAAG,CAAC,CAAC,CAAChB,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAACyC,QAAQG,SAAS,IAAI/C,OAAO0B,OAAO,CAACpC,sBAA0D;QACxG,IAAIwD,MAAM1C,GAAG,CAACwC,WAAW,CAACE,MAAM1C,GAAG,CAAC2C,WAAW;YAC7C,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,cAAc,EAAEwB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAE3B,KAAK,mBAAmB,EAAEwB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAe5B,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAM6B,SAAS3C;IACf,MAAMkB,UAAUxB,OAAOC,IAAI,CAACgD,QAAQ/C,MAAM,CAAC,CAACC,IAAM,CAACT,kBAAkBU,GAAG,CAACD;IACzE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAAChB,sBAAsB4C,OAAOjC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI6B,OAAOC,OAAO,KAAKpB,aAAa,CAACzB,sBAAsB4C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI6B,OAAOE,OAAO,KAAKrB,WAAWa,sBAAsBvB,MAAM6B,OAAOE,OAAO,EAAErC;IAC9E,IAAImC,OAAO9C,CAAC,KAAK2B,aAAc,CAAA,OAAOmB,OAAO9C,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACU,OAAO9C,CAAC,KAAK8C,OAAO9C,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAI6B,OAAOG,KAAK,KAAKtB,aAAa,OAAOmB,OAAOG,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAASiC,mBAAmBjC,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IAC1E,2FAA2F;IAC3F,yDAAyD;IACzD,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uCAAuC,EAAEkC,KAAKC,SAAS,CAACjD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMkD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGlD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMqC,QAAQnD;IACd,IAAI,SAASmD,OAAO;QAClB,MAAMjC,UAAUxB,OAAOC,IAAI,CAACwD,OAAOvD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIqB,QAAQf,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOoC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAIvE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYqC,OAAO;QACrB,MAAMjC,UAAUxB,OAAOC,IAAI,CAACwD,OAAOvD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOoC,MAAMG,MAAM,KAAK,YAAYH,MAAMG,MAAM,CAACD,IAAI,OAAO,IAAI;YAClE,MAAM,IAAIvE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIqC,MAAMR,MAAM,KAAKnB,aAAc,CAAA,OAAO2B,MAAMR,MAAM,KAAK,YAAYQ,MAAMR,MAAM,CAACxC,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIqC,MAAMzC,OAAO,KAAKc,aAAa,CAACzB,sBAAsBoD,MAAMzC,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIqC,MAAMP,OAAO,KAAKpB,aAAa,CAACzB,sBAAsBoD,MAAMP,OAAO,GAAG;YACxE,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIqC,MAAML,KAAK,KAAKtB,aAAa,OAAO2B,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIqC,MAAMtD,CAAC,KAAK2B,aAAc,CAAA,OAAO2B,MAAMtD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACkB,MAAMtD,CAAC,KAAKsD,MAAMtD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIhC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASyC,eAAehD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAI+D,MAAM,KAAKhC,WAAW;QAC5B,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,yFAAyF;IACzF,uFAAuF;IACvF,4FAA4F;IAC5F,IAAIf,IAAIgE,OAAO,KAAKjC,WAAW;QAC7B,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,uHAAuH,CAAC;IAC/K;IAEA,MAAMkD,UAAUjE,IAAIiE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYzD,MAAMC,OAAO,CAACwD,YAAYhE,OAAOC,IAAI,CAAC+D,SAASvD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACM,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACsC,SAAU;QACnDhB,eAAe5B,MAAMd,OAAOQ;IAC9B;IACA,IAAIkD,QAAQC,OAAO,KAAKnC,WAAW;QACjC,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAImE,OAAO,KAAKpC,WAAW/B,IAAImE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOnE,IAAImE,OAAO,KAAK,YAAYnE,IAAImE,OAAO,KAAK,QAAQ3D,MAAMC,OAAO,CAACT,IAAImE,OAAO,GAAG;QACzF,MAAM,IAAI9E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMoD,UAAUnE,IAAImE,OAAO;IAC3B,KAAK,MAAM,CAAC9C,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACwC,SAAU;QACnDb,mBAAmBjC,MAAMd,OAAOQ;IAClC;IAEA,IAAIf,IAAIoE,QAAQ,KAAKrC,WAAW;QAC9BR,sBAAsBvB,IAAIoE,QAAQ,EAAErD;IACtC;IACA,IAAIf,IAAI6B,KAAK,KAAKE,WAAW;QAC3BH,mBAAmB5B,IAAI6B,KAAK,EAAEd;IAChC;IACA,IAAIf,IAAIqE,KAAK,KAAKtC,WAAW;QAC3Bb,iBAAiBlB,IAAIqE,KAAK,EAAEtD;IAC9B;IAEA,8FAA8F;IAC9F,IAAIf,IAAI6B,KAAK,KAAKE,WAAW;QAC3B,KAAK,MAAM,CAACV,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACsC,SAAU;YACnD,MAAMb,UAAU,AAAC7C,MAAkC6C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAI/D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAOrB;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport { type Config, STORE_NAMES, type StoreName } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'store', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\n// The one source of truth for what the embed block accepts. schema.json carries an editor-facing\n// copy for autocomplete; nothing at runtime reads it.\nexport const KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages', 'chunkTokens']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\nfunction validateStoreKey(value: unknown, configPath: string): void {\n  if (!STORE_NAMES.includes(value as StoreName)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: store must be ${STORE_NAMES.map((name) => `\"${name}\"`).join(' or ')}`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n  if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that doesn't\n// need the top-level embed block: links is seeded by word-match rows, so declaring it without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string is rejected rather than inferred to be SQL: an entry says which of the two\n  // verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  // `content` (content.tokenize) is rejected by name for the same reason: every outcome it\n  // selected (unspaced-script search, stemming, quoted-phrase substring matching) is now\n  // automatic on every store, so a config that still sets it would silently get nothing back.\n  if (cfg.content !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize was removed -- every outcome it selected is now automatic on every store; remove the \"content\" block`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.store !== undefined) {\n    validateStoreKey(cfg.store, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","STORE_NAMES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateStoreKey","includes","map","name","join","validateFeaturesBlock","block","unknown","v","entries","validateEmbedBlock","embed","model","undefined","provider","url","key","languages","some","l","chunkTokens","Number","isInteger","isSignalName","isSignalWeight","isFinite","validatePresetSignals","signal","weight","names","requires","validatePreset","preset","exclude","signals","where","validateSavedQuery","JSON","stringify","slice","entry","sql","trim","search","validateConfig","checks","content","presets","default","queries","features","store"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AACnF,SAAsBC,WAAW,QAAwB,aAAa;AAEtE,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAS;CAAU;AACrG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,iGAAiG;AACjG,sDAAsD;AACtD,OAAO,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;IAAa;CAAc,EAAE;AACzG,MAAMI,oBAAoB,IAAIJ,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASK,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACX,WAAWY,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,SAASG,iBAAiBX,KAAc,EAAEQ,UAAkB;IAC1D,IAAI,CAACvB,YAAY2B,QAAQ,CAACZ,QAAqB;QAC7C,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,gBAAgB,EAAEvB,YAAY4B,GAAG,CAAC,CAACC,OAAS,CAAC,CAAC,EAAEA,KAAK,CAAC,CAAC,EAAEC,IAAI,CAAC,SAAS;IAC9H;AACF;AAEA,SAASC,sBAAsBhB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMS,QAAQjB;IACd,MAAMkB,UAAUxB,OAAOC,IAAI,CAACsB,OAAOrB,MAAM,CAAC,CAACC,IAAM,CAACR,mBAAmBS,GAAG,CAACD;IACzE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEU,QAAQH,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACD,MAAMK,EAAE,IAAIzB,OAAO0B,OAAO,CAACH,OAAQ;QAC7C,IAAI,OAAOE,MAAM,WAAW;YAC1B,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEM,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASO,mBAAmBrB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMc,QAAQtB;IACd,MAAMkB,UAAUxB,OAAOC,IAAI,CAAC2B,OAAO1B,MAAM,CAAC,CAACC,IAAM,CAACP,iBAAiBQ,GAAG,CAACD;IACvE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEU,QAAQH,IAAI,CAAC,OAAO;IACxG;IACA,IAAIO,MAAMC,KAAK,KAAKC,aAAa,OAAOF,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAIzC,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIc,MAAMC,KAAK,KAAKC,aAAaF,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAIzC,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIc,MAAMG,QAAQ,KAAKD,aAAaF,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIc,MAAMI,GAAG,KAAKF,aAAa,OAAOF,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIc,MAAMK,GAAG,KAAKH,aAAa,OAAOF,MAAMK,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIc,MAAMG,QAAQ,KAAK,YAAaH,CAAAA,MAAMI,GAAG,KAAKF,aAAaF,MAAMI,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIc,MAAMM,SAAS,KAAKJ,aAAc,CAAA,CAACvB,MAAMC,OAAO,CAACoB,MAAMM,SAAS,KAAKN,MAAMM,SAAS,CAACzB,MAAM,KAAK,KAAKmB,MAAMM,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;IACA,IAAIc,MAAMS,WAAW,KAAKP,aAAc,CAAA,OAAOF,MAAMS,WAAW,KAAK,YAAY,CAACC,OAAOC,SAAS,CAACX,MAAMS,WAAW,KAAKT,MAAMS,WAAW,IAAI,CAAA,GAAI;QAChJ,MAAM,IAAIjD,WAAW,kBAAkB,GAAG0B,WAAW,8CAA8C,CAAC;IACtG;AACF;AAEA,SAAS0B,aAAalC,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmC6B,QAAQ,CAACZ;AACnF;AAEA,SAASmC,eAAenC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYgC,OAAOI,QAAQ,CAACpC,UAAUA,QAAQ;AACxE;AAEA,iGAAiG;AACjG,qHAAqH;AACrH,SAASqC,sBAAsBvB,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEM,KAAK,2FAA2F,EAAE/B,aAAagC,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMK,UAAU1B,OAAO0B,OAAO,CAACpB;IAC/B,MAAMkB,UAAUE,QAAQxB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACqC,aAAarC,IAAIgB,GAAG,CAAC,CAAC,CAAChB,EAAE,GAAKA;IACvE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,iCAAiC,EAAEI,QAAQH,IAAI,CAAC,MAAM,oBAAoB,EAAEhC,aAAagC,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAACuB,QAAQC,OAAO,IAAInB,QAAS;QACtC,IAAI,CAACe,eAAeI,SAAS;YAC3B,MAAM,IAAIzD,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,SAAS,EAAEwB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAIrD,IAAIiC,QAAQP,GAAG,CAAC,CAAC,CAAChB,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAACyC,QAAQG,SAAS,IAAI/C,OAAO0B,OAAO,CAACpC,sBAA0D;QACxG,IAAIwD,MAAM1C,GAAG,CAACwC,WAAW,CAACE,MAAM1C,GAAG,CAAC2C,WAAW;YAC7C,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,cAAc,EAAEwB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAE3B,KAAK,mBAAmB,EAAEwB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAe5B,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAM6B,SAAS3C;IACf,MAAMkB,UAAUxB,OAAOC,IAAI,CAACgD,QAAQ/C,MAAM,CAAC,CAACC,IAAM,CAACT,kBAAkBU,GAAG,CAACD;IACzE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAAChB,sBAAsB4C,OAAOjC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI6B,OAAOC,OAAO,KAAKpB,aAAa,CAACzB,sBAAsB4C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI6B,OAAOE,OAAO,KAAKrB,WAAWa,sBAAsBvB,MAAM6B,OAAOE,OAAO,EAAErC;IAC9E,IAAImC,OAAO9C,CAAC,KAAK2B,aAAc,CAAA,OAAOmB,OAAO9C,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACU,OAAO9C,CAAC,KAAK8C,OAAO9C,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAI6B,OAAOG,KAAK,KAAKtB,aAAa,OAAOmB,OAAOG,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAASiC,mBAAmBjC,IAAY,EAAEd,KAAc,EAAEQ,UAAkB;IAC1E,2FAA2F;IAC3F,yDAAyD;IACzD,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uCAAuC,EAAEkC,KAAKC,SAAS,CAACjD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMkD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGlD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMqC,QAAQnD;IACd,IAAI,SAASmD,OAAO;QAClB,MAAMjC,UAAUxB,OAAOC,IAAI,CAACwD,OAAOvD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIqB,QAAQf,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOoC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAIvE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYqC,OAAO;QACrB,MAAMjC,UAAUxB,OAAOC,IAAI,CAACwD,OAAOvD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIqB,QAAQf,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,oBAAoB,EAAEI,QAAQH,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOoC,MAAMG,MAAM,KAAK,YAAYH,MAAMG,MAAM,CAACD,IAAI,OAAO,IAAI;YAClE,MAAM,IAAIvE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIqC,MAAMR,MAAM,KAAKnB,aAAc,CAAA,OAAO2B,MAAMR,MAAM,KAAK,YAAYQ,MAAMR,MAAM,CAACxC,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIqC,MAAMzC,OAAO,KAAKc,aAAa,CAACzB,sBAAsBoD,MAAMzC,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIqC,MAAMP,OAAO,KAAKpB,aAAa,CAACzB,sBAAsBoD,MAAMP,OAAO,GAAG;YACxE,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIqC,MAAML,KAAK,KAAKtB,aAAa,OAAO2B,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIqC,MAAMtD,CAAC,KAAK2B,aAAc,CAAA,OAAO2B,MAAMtD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACkB,MAAMtD,CAAC,KAAKsD,MAAMtD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIhC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASyC,eAAehD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAI+D,MAAM,KAAKhC,WAAW;QAC5B,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,yFAAyF;IACzF,uFAAuF;IACvF,4FAA4F;IAC5F,IAAIf,IAAIgE,OAAO,KAAKjC,WAAW;QAC7B,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,uHAAuH,CAAC;IAC/K;IAEA,MAAMkD,UAAUjE,IAAIiE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYzD,MAAMC,OAAO,CAACwD,YAAYhE,OAAOC,IAAI,CAAC+D,SAASvD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACM,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACsC,SAAU;QACnDhB,eAAe5B,MAAMd,OAAOQ;IAC9B;IACA,IAAIkD,QAAQC,OAAO,KAAKnC,WAAW;QACjC,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAImE,OAAO,KAAKpC,WAAW/B,IAAImE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOnE,IAAImE,OAAO,KAAK,YAAYnE,IAAImE,OAAO,KAAK,QAAQ3D,MAAMC,OAAO,CAACT,IAAImE,OAAO,GAAG;QACzF,MAAM,IAAI9E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMoD,UAAUnE,IAAImE,OAAO;IAC3B,KAAK,MAAM,CAAC9C,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACwC,SAAU;QACnDb,mBAAmBjC,MAAMd,OAAOQ;IAClC;IAEA,IAAIf,IAAIoE,QAAQ,KAAKrC,WAAW;QAC9BR,sBAAsBvB,IAAIoE,QAAQ,EAAErD;IACtC;IACA,IAAIf,IAAI6B,KAAK,KAAKE,WAAW;QAC3BH,mBAAmB5B,IAAI6B,KAAK,EAAEd;IAChC;IACA,IAAIf,IAAIqE,KAAK,KAAKtC,WAAW;QAC3Bb,iBAAiBlB,IAAIqE,KAAK,EAAEtD;IAC9B;IAEA,8FAA8F;IAC9F,IAAIf,IAAI6B,KAAK,KAAKE,WAAW;QAC3B,KAAK,MAAM,CAACV,MAAMd,MAAM,IAAIN,OAAO0B,OAAO,CAACsC,SAAU;YACnD,MAAMb,UAAU,AAAC7C,MAAkC6C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAI/D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEM,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAOrB;AACT"}

package/dist/esm/index.d.ts

@@ -6,7 +6,7 @@ export type { SenseErrorCode } from './errors.js';
 export { SenseError } from './errors.js';
 export type { Format, Row, RowFormat } from './output/output.js';
 export { printRows } from './output/output.js';
-export type { OpenResult } from './store/index.js';
+export type { OpenResult, Stages } from './store/index.js';
 export { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.js';
 export type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.js';
 export type { WatchEvent, WatchOptions } from './watch.js';

package/dist/esm/index.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAInF,SAASC,QAAQ,QAAQ,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/index.ts"],"sourcesContent":["// Public library API. Deliberately small: every export is a stability promise;\n// internals (feature registry, graph, scan, meta) stay module-private.\n\nexport type { Peek, PresetCoverage, SearchOptions, TreeMap } from './commands/index.ts';\nexport { mapTree, peek, presetCoverage, search } from './commands/index.ts';\nexport type { Config, EmbedConfig, FeatureName, InitOverrides, Preset, ResolvedConfig, SavedQuery, SavedSearch, SearchOverrides, SignalName, SignalWeights, StoreName } from './config/index.ts';\nexport { CONFIG_FILENAME, initConfig, loadConfig, migrateConfig, SIGNAL_NAMES, STATE_DIR, STORE_NAMES, SUPPORTED_CONFIG_VERSION } from './config/index.ts';\nexport type { SenseErrorCode } from './errors.ts';\nexport { SenseError } from './errors.ts';\nexport type { Format, Row, RowFormat } from './output/output.ts';\nexport { printRows } from './output/output.ts';\nexport type { OpenResult, Stages } from './store/index.ts';\nexport { CAPABILITY_NAMES, clearCache, openStore as open } from './store/index.ts';\nexport type { Capability, DocumentStore, FieldStat, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './store/types.ts';\n\nexport type { WatchEvent, WatchOptions } from './watch.ts';\nexport { runWatch } from './watch.ts';\n"],"names":["mapTree","peek","presetCoverage","search","CONFIG_FILENAME","initConfig","loadConfig","migrateConfig","SIGNAL_NAMES","STATE_DIR","STORE_NAMES","SUPPORTED_CONFIG_VERSION","SenseError","printRows","CAPABILITY_NAMES","clearCache","openStore","open","runWatch"],"mappings":"AAAA,+EAA+E;AAC/E,uEAAuE;AAGvE,SAASA,OAAO,EAAEC,IAAI,EAAEC,cAAc,EAAEC,MAAM,QAAQ,sBAAsB;AAE5E,SAASC,eAAe,EAAEC,UAAU,EAAEC,UAAU,EAAEC,aAAa,EAAEC,YAAY,EAAEC,SAAS,EAAEC,WAAW,EAAEC,wBAAwB,QAAQ,oBAAoB;AAE3J,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,SAAS,QAAQ,qBAAqB;AAE/C,SAASC,gBAAgB,EAAEC,UAAU,EAAEC,aAAaC,IAAI,QAAQ,mBAAmB;AAInF,SAASC,QAAQ,QAAQ,aAAa"}

package/dist/esm/scan/pool.d.ts

@@ -5,6 +5,7 @@ import type { FileStat } from './list.js';
 export type FileResult = {
     doc: ParsedDoc;
     warnings: string[];
+    parseMs?: number;
 };
 export declare class ParsePool {
     private pool;

package/dist/esm/scan/pool.js

@@ -58,7 +58,8 @@ export class ParsePool {
             onParsed === null || onParsed === void 0 ? void 0 : onParsed(++done);
             return {
                 doc: result.doc,
-                warnings: result.warnings
+                warnings: result.warnings,
+                parseMs: result.parseMs
             };
         }));
     }

package/dist/esm/scan/pool.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\nexport type FileResult = { doc: ParsedDoc; warnings: string[] };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","dirname","join","reviveError","_require","require","url","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","done","Promise","all","file","result","ok","error","doc","warnings","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAS1C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOT,cAAc,YAAYM,GAAG;IAC1C,KAAK,MAAMI,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaL,KAAKD,QAAQQ,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAIA,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGf,SAAS;YAC9B,IAAI,CAACc,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUZ;gBAAqBa,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YACrH,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQpB,IAAI;QAAE;QACjG,MAAMO,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIe,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBT,MAAMK,GAAG,CAAC,OAAOK;YACf,MAAMC,SAAU,MAAMlB,KAAKM,GAAG,CAACW;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAMlC,YAAYiC,OAAOE,KAAK;YAC9CV,qBAAAA,+BAAAA,SAAW,EAAEI;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;YAAC;QACtD;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACvB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGwB;QACZ,MAAMxB,KAAKyB,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGpB,eAAe;;AAsCjB"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/pool.ts"],"sourcesContent":["import { createRequire } from 'node:module';\nimport { dirname, join } from 'node:path';\n// Type-only: erased at build, but keeps depcheck's usage check satisfied for the tier-2\n// `_require` below (see coding-standards' deferral tiers).\nimport type * as TinypoolNS from 'tinypool';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from '../scan/index.ts';\nimport type { ParseTask, ParseTaskResult, ParseWorkerData } from '../workers/parse.ts';\nimport type { FileStat } from './list.ts';\nimport { reviveError } from './worker-error.ts';\n\n// Tinypool is ESM-only; our floor (>=22.20) has native require(esm), so the tier-2 house deferral reaches it.\nconst _require = typeof require === 'undefined' ? createRequire(import.meta.url) : require;\n\n// Worker MUST load from dist/cjs/: a worker_threads thread is a fresh realm with no inherited\n// TS loader hook, so a source path cannot work. Resolved on first pooled dispatch, not at import.\nlet workerFile: string | undefined;\nfunction resolveWorkerFile(): string {\n  if (workerFile) return workerFile;\n  const load = createRequire(import.meta.url);\n  for (const rel of ['..', '../..', '../../..']) {\n    try {\n      if ((load(`${rel}/package.json`) as { name?: string }).name === 'sensemaking') {\n        workerFile = join(dirname(load.resolve(`${rel}/package.json`)), 'dist', 'cjs', 'workers', 'parse.js');\n        return workerFile;\n      }\n    } catch {}\n  }\n  throw new Error('cannot locate the sensemaking package root, so the parse worker cannot be found; run npm run build');\n}\n\n// parseMs is worker-side only (see workers/parse.ts); absent on the serial path.\nexport type FileResult = { doc: ParsedDoc; warnings: string[]; parseMs?: number };\n\n// One Tinypool per instance, created lazily on first dispatch and reused by every later call:\n// a builder owns one of these for its whole lifetime instead of paying pool startup per reconcile.\nexport class ParsePool {\n  private pool: TinypoolNS.Tinypool | undefined;\n  // Tinypools this instance has constructed. One dispatch or a hundred should leave it at 1;\n  // it is how a caller, and the specs, observe that a lifetime reuses its pool rather than churning one.\n  poolsCreated = 0;\n\n  private ensure(workerData: ParseWorkerData, maxWorkers: number): TinypoolNS.Tinypool {\n    if (!this.pool) {\n      const { Tinypool } = _require('tinypool') as typeof TinypoolNS;\n      this.pool = new Tinypool({ filename: resolveWorkerFile(), minThreads: maxWorkers, maxThreads: maxWorkers, workerData });\n      this.poolsCreated++;\n    }\n    return this.pool;\n  }\n\n  // Never the tree: a worker task carries one FileStat and returns only what parseFile returns --\n  // extracted text and feature values, never the mdast tree.\n  async run(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number): Promise<FileResult[]> {\n    // cfg and the feature selection are constant for the dispatch, so they cross once per worker\n    // as workerData. Features carry closures and cannot cross at all; only their names do.\n    const workerData: ParseWorkerData = { cfg, featureNames: features.map((feature) => feature.name) };\n    const pool = this.ensure(workerData, maxWorkers);\n    let done = 0;\n    // Promise.all over a mapped array is load-bearing: the resolved array keeps `files` order\n    // regardless of task completion order, which first-seen column order depends on downstream.\n    return Promise.all(\n      files.map(async (file): Promise<FileResult> => {\n        const result = (await pool.run(file as ParseTask)) as ParseTaskResult;\n        if (!result.ok) throw reviveError(result.error);\n        onParsed?.(++done);\n        return { doc: result.doc, warnings: result.warnings, parseMs: result.parseMs };\n      })\n    );\n  }\n\n  // A pool never created costs nothing to destroy.\n  async close(): Promise<void> {\n    if (!this.pool) return;\n    const pool = this.pool;\n    this.pool = undefined;\n    await pool.destroy();\n  }\n}\n"],"names":["createRequire","dirname","join","reviveError","_require","require","url","workerFile","resolveWorkerFile","load","rel","name","resolve","Error","ParsePool","ensure","workerData","maxWorkers","pool","Tinypool","filename","minThreads","maxThreads","poolsCreated","run","files","features","cfg","onParsed","featureNames","map","feature","done","Promise","all","file","result","ok","error","doc","warnings","parseMs","close","undefined","destroy"],"mappings":"AAAA,SAASA,aAAa,QAAQ,cAAc;AAC5C,SAASC,OAAO,EAAEC,IAAI,QAAQ,YAAY;AAS1C,SAASC,WAAW,QAAQ,oBAAoB;AAEhD,8GAA8G;AAC9G,MAAMC,WAAW,OAAOC,YAAY,cAAcL,cAAc,YAAYM,GAAG,IAAID;AAEnF,8FAA8F;AAC9F,kGAAkG;AAClG,IAAIE;AACJ,SAASC;IACP,IAAID,YAAY,OAAOA;IACvB,MAAME,OAAOT,cAAc,YAAYM,GAAG;IAC1C,KAAK,MAAMI,OAAO;QAAC;QAAM;QAAS;KAAW,CAAE;QAC7C,IAAI;YACF,IAAI,AAACD,KAAK,GAAGC,IAAI,aAAa,CAAC,EAAwBC,IAAI,KAAK,eAAe;gBAC7EJ,aAAaL,KAAKD,QAAQQ,KAAKG,OAAO,CAAC,GAAGF,IAAI,aAAa,CAAC,IAAI,QAAQ,OAAO,WAAW;gBAC1F,OAAOH;YACT;QACF,EAAE,OAAM,CAAC;IACX;IACA,MAAM,IAAIM,MAAM;AAClB;AAKA,8FAA8F;AAC9F,mGAAmG;AACnG,OAAO,MAAMC;IAMHC,OAAOC,UAA2B,EAAEC,UAAkB,EAAuB;QACnF,IAAI,CAAC,IAAI,CAACC,IAAI,EAAE;YACd,MAAM,EAAEC,QAAQ,EAAE,GAAGf,SAAS;YAC9B,IAAI,CAACc,IAAI,GAAG,IAAIC,SAAS;gBAAEC,UAAUZ;gBAAqBa,YAAYJ;gBAAYK,YAAYL;gBAAYD;YAAW;YACrH,IAAI,CAACO,YAAY;QACnB;QACA,OAAO,IAAI,CAACL,IAAI;IAClB;IAEA,gGAAgG;IAChG,2DAA2D;IAC3D,MAAMM,IAAIC,KAAiB,EAAEC,QAAmB,EAAEC,GAAW,EAAEC,QAA8C,EAAEX,UAAkB,EAAyB;QACxJ,6FAA6F;QAC7F,uFAAuF;QACvF,MAAMD,aAA8B;YAAEW;YAAKE,cAAcH,SAASI,GAAG,CAAC,CAACC,UAAYA,QAAQpB,IAAI;QAAE;QACjG,MAAMO,OAAO,IAAI,CAACH,MAAM,CAACC,YAAYC;QACrC,IAAIe,OAAO;QACX,0FAA0F;QAC1F,4FAA4F;QAC5F,OAAOC,QAAQC,GAAG,CAChBT,MAAMK,GAAG,CAAC,OAAOK;YACf,MAAMC,SAAU,MAAMlB,KAAKM,GAAG,CAACW;YAC/B,IAAI,CAACC,OAAOC,EAAE,EAAE,MAAMlC,YAAYiC,OAAOE,KAAK;YAC9CV,qBAAAA,+BAAAA,SAAW,EAAEI;YACb,OAAO;gBAAEO,KAAKH,OAAOG,GAAG;gBAAEC,UAAUJ,OAAOI,QAAQ;gBAAEC,SAASL,OAAOK,OAAO;YAAC;QAC/E;IAEJ;IAEA,iDAAiD;IACjD,MAAMC,QAAuB;QAC3B,IAAI,CAAC,IAAI,CAACxB,IAAI,EAAE;QAChB,MAAMA,OAAO,IAAI,CAACA,IAAI;QACtB,IAAI,CAACA,IAAI,GAAGyB;QACZ,MAAMzB,KAAK0B,OAAO;IACpB;;QAvCA,2FAA2F;QAC3F,uGAAuG;aACvGrB,eAAe;;AAsCjB"}

package/dist/esm/scan/reparse.d.ts

@@ -7,6 +7,7 @@ export interface ReparseResult {
     docs: ParsedDoc[];
     warnings: string[];
     newColumns: string[];
+    workerParseMs: number;
 }
 export interface ReparseOptions {
     threshold?: number;

package/dist/esm/scan/reparse.js

@@ -47,8 +47,10 @@ export async function reparseFiles(files, features, cfg, knownColumns, onParsed,
     const warnings = [];
     const newColumns = [];
     const seen = new Set(knownColumns);
-    for (const { doc, warnings: fileWarnings } of results){
+    let workerParseMs = 0;
+    for (const { doc, warnings: fileWarnings, parseMs } of results){
         warnings.push(...fileWarnings);
+        workerParseMs += parseMs !== null && parseMs !== void 0 ? parseMs : 0;
         for (const key of Object.keys(doc.data)){
             if (!seen.has(key)) {
                 seen.add(key);
@@ -60,6 +62,7 @@ export async function reparseFiles(files, features, cfg, knownColumns, onParsed,
     return {
         docs,
         warnings,
-        newColumns
+        newColumns,
+        workerParseMs
     };
 }

package/dist/esm/scan/reparse.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  for (const { doc, warnings: fileWarnings } of results) {\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns };\n}\n"],"names":["availableParallelism","parseFile","ParsePool","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","pool","run","ephemeral","close","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","warnings","newColumns","seen","Set","doc","fileWarnings","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,oBAAoB,QAAQ,UAAU;AAI/C,SAASC,SAAS,QAAQ,aAAa;AAGvC,SAASC,SAAS,QAAQ,YAAY;AAsBtC,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAMC,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAACjB,UAAUO,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB,EAAEC,IAA2B;IAC/K,IAAIA,MAAM,OAAOA,KAAKC,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAC1D,MAAMG,YAAY,IAAIrB;IACtB,IAAI;QACF,MAAMa,UAAU,MAAMQ,UAAUD,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;QACpE,MAAMG,UAAUC,KAAK;QACrB,OAAOT;IACT,EAAE,OAAOU,KAAK;QACZ,oFAAoF;QACpF,8DAA8D;QAC9D,MAAMF,UAAUC,KAAK,GAAGE,KAAK,CAAC,KAAO;QACrC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAad,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEqB,YAAiC,EAAEd,QAAiC,EAAEe,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;IACvC,MAAMiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAC5B,qBAAqBJ;IACvE,MAAMe,UAAUF,MAAMoB,MAAM,IAAIH,YAAY,MAAMX,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,YAAYS,QAAQR,IAAI,IAAIT,cAAcC,OAAOP,UAAUC,KAAKO;IAEhK,MAAMoB,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIV;IACrB,KAAK,MAAM,EAAEW,GAAG,EAAEJ,UAAUK,YAAY,EAAE,IAAIzB,QAAS;QACrDoB,SAASjB,IAAI,IAAIsB;QACjB,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACJ,IAAIK,IAAI,EAAG;YACvC,IAAI,CAACP,KAAKQ,GAAG,CAACJ,MAAM;gBAClBJ,KAAKS,GAAG,CAACL;gBACTL,WAAWlB,IAAI,CAACuB;YAClB;QACF;QACAP,KAAKhB,IAAI,CAACqB;IACZ;IAEA,OAAO;QAAEL;QAAMC;QAAUC;IAAW;AACtC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/scan/reparse.ts"],"sourcesContent":["import { availableParallelism } from 'node:os';\nimport type { Config } from '../config/index.ts';\nimport type { Feature } from '../features/types.ts';\nimport type { ParsedDoc } from './index.ts';\nimport { parseFile } from './index.ts';\nimport type { FileStat } from './list.ts';\nimport type { FileResult } from './pool.ts';\nimport { ParsePool } from './pool.ts';\n\n// Store-agnostic per-file parse pass shared by every store's reconcile(). Index-preserving:\n// one pass over `files`, pushed in order.\nexport interface ReparseResult {\n  docs: ParsedDoc[];\n  warnings: string[];\n  // Frontmatter keys not in `knownColumns`, first-seen order -- the order callers ALTER TABLE ADD COLUMN in.\n  newColumns: string[];\n  // Sum of worker-side parseFile time (workers/parse.ts) across files; 0 on the serial path.\n  workerParseMs: number;\n}\n\nexport interface ReparseOptions {\n  // Overrides DEFAULT_THRESHOLD below, so tests can force either dispatch mode without\n  // needing thousands of fixture files. Internal: no caller in src/ passes it.\n  threshold?: number;\n  // Overrides DEFAULT_MAX_WORKERS below.\n  maxWorkers?: number;\n  // A builder-owned pool to dispatch through instead of creating and destroying one for this\n  // call alone (see store/builder.ts). Absent, reparseFiles manages its own ephemeral pool.\n  pool?: ParsePool;\n}\n\n// Measured 2026-08-29 on a 14-core Apple M4 Pro: serial and pooled cross between 160-180 files\n// (pool fixed cost ~85-100ms), so 200 keeps margin. 8 workers ties 10 in a 6,566-file sweep; 12+ trends worse.\nconst DEFAULT_THRESHOLD = 200;\nconst DEFAULT_MAX_WORKERS = 8;\n\n// Per-file feature filtering, shared by the serial loop (reads `features` from the caller) and\n// the worker path (can only pass `cfg` across the thread boundary, so derives `features` itself).\nexport function featuresForFile(features: Feature[], cfg: Config, file: FileStat): Feature[] {\n  return features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n}\n\nfunction reparseSerial(files: FileStat[], features: Feature[], cfg: Config, onParsed?: (done: number) => void): FileResult[] {\n  const results: FileResult[] = [];\n  let done = 0;\n  for (const file of files) {\n    const fileFeatures = featuresForFile(features, cfg, file);\n    results.push(parseFile(file, fileFeatures, cfg));\n    onParsed?.(++done);\n  }\n  return results;\n}\n\n// Dispatches through `pool` when the caller owns one (a builder, kept alive across calls); with\n// none given, opens an ephemeral ParsePool for this call alone and always closes it.\nasync function reparsePooled(files: FileStat[], features: Feature[], cfg: Config, onParsed: ((done: number) => void) | undefined, maxWorkers: number, pool: ParsePool | undefined): Promise<FileResult[]> {\n  if (pool) return pool.run(files, features, cfg, onParsed, maxWorkers);\n  const ephemeral = new ParsePool();\n  try {\n    const results = await ephemeral.run(files, features, cfg, onParsed, maxWorkers);\n    await ephemeral.close();\n    return results;\n  } catch (err) {\n    // Tear the pool down before rethrowing, but a close failure must not mask the parse\n    // failure that caused it -- not a `finally`, for that reason.\n    await ephemeral.close().catch(() => {});\n    throw err;\n  }\n}\n\n// `onParsed` receives the running count (1-based) after each file, mirroring a Progress.tick\n// call; pass one in to keep progress reporting working without this module owning a reporter.\nexport async function reparseFiles(files: FileStat[], features: Feature[], cfg: Config, knownColumns: ReadonlySet<string>, onParsed?: (done: number) => void, options: ReparseOptions = {}): Promise<ReparseResult> {\n  const threshold = options.threshold ?? DEFAULT_THRESHOLD;\n  const maxWorkers = options.maxWorkers ?? Math.min(DEFAULT_MAX_WORKERS, availableParallelism());\n  const results = files.length >= threshold ? await reparsePooled(files, features, cfg, onParsed, maxWorkers, options.pool) : reparseSerial(files, features, cfg, onParsed);\n\n  const docs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n  const newColumns: string[] = [];\n  const seen = new Set(knownColumns);\n  let workerParseMs = 0;\n  for (const { doc, warnings: fileWarnings, parseMs } of results) {\n    warnings.push(...fileWarnings);\n    workerParseMs += parseMs ?? 0;\n    for (const key of Object.keys(doc.data)) {\n      if (!seen.has(key)) {\n        seen.add(key);\n        newColumns.push(key);\n      }\n    }\n    docs.push(doc);\n  }\n\n  return { docs, warnings, newColumns, workerParseMs };\n}\n"],"names":["availableParallelism","parseFile","ParsePool","DEFAULT_THRESHOLD","DEFAULT_MAX_WORKERS","featuresForFile","features","cfg","file","filter","feature","enabledForFile","reparseSerial","files","onParsed","results","done","fileFeatures","push","reparsePooled","maxWorkers","pool","run","ephemeral","close","err","catch","reparseFiles","knownColumns","options","threshold","Math","min","length","docs","warnings","newColumns","seen","Set","workerParseMs","doc","fileWarnings","parseMs","key","Object","keys","data","has","add"],"mappings":"AAAA,SAASA,oBAAoB,QAAQ,UAAU;AAI/C,SAASC,SAAS,QAAQ,aAAa;AAGvC,SAASC,SAAS,QAAQ,YAAY;AAwBtC,+FAA+F;AAC/F,+GAA+G;AAC/G,MAAMC,oBAAoB;AAC1B,MAAMC,sBAAsB;AAE5B,+FAA+F;AAC/F,kGAAkG;AAClG,OAAO,SAASC,gBAAgBC,QAAmB,EAAEC,GAAW,EAAEC,IAAc;IAC9E,OAAOF,SAASG,MAAM,CAAC,CAACC,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACJ,KAAKC;AAC7F;AAEA,SAASI,cAAcC,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAAiC;IAC3G,MAAMC,UAAwB,EAAE;IAChC,IAAIC,OAAO;IACX,KAAK,MAAMR,QAAQK,MAAO;QACxB,MAAMI,eAAeZ,gBAAgBC,UAAUC,KAAKC;QACpDO,QAAQG,IAAI,CAACjB,UAAUO,MAAMS,cAAcV;QAC3CO,qBAAAA,+BAAAA,SAAW,EAAEE;IACf;IACA,OAAOD;AACT;AAEA,gGAAgG;AAChG,qFAAqF;AACrF,eAAeI,cAAcN,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEO,QAA8C,EAAEM,UAAkB,EAAEC,IAA2B;IAC/K,IAAIA,MAAM,OAAOA,KAAKC,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;IAC1D,MAAMG,YAAY,IAAIrB;IACtB,IAAI;QACF,MAAMa,UAAU,MAAMQ,UAAUD,GAAG,CAACT,OAAOP,UAAUC,KAAKO,UAAUM;QACpE,MAAMG,UAAUC,KAAK;QACrB,OAAOT;IACT,EAAE,OAAOU,KAAK;QACZ,oFAAoF;QACpF,8DAA8D;QAC9D,MAAMF,UAAUC,KAAK,GAAGE,KAAK,CAAC,KAAO;QACrC,MAAMD;IACR;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeE,aAAad,KAAiB,EAAEP,QAAmB,EAAEC,GAAW,EAAEqB,YAAiC,EAAEd,QAAiC,EAAEe,UAA0B,CAAC,CAAC;QACtKA,oBACCA;IADnB,MAAMC,aAAYD,qBAAAA,QAAQC,SAAS,cAAjBD,gCAAAA,qBAAqB1B;IACvC,MAAMiB,cAAaS,sBAAAA,QAAQT,UAAU,cAAlBS,iCAAAA,sBAAsBE,KAAKC,GAAG,CAAC5B,qBAAqBJ;IACvE,MAAMe,UAAUF,MAAMoB,MAAM,IAAIH,YAAY,MAAMX,cAAcN,OAAOP,UAAUC,KAAKO,UAAUM,YAAYS,QAAQR,IAAI,IAAIT,cAAcC,OAAOP,UAAUC,KAAKO;IAEhK,MAAMoB,OAAoB,EAAE;IAC5B,MAAMC,WAAqB,EAAE;IAC7B,MAAMC,aAAuB,EAAE;IAC/B,MAAMC,OAAO,IAAIC,IAAIV;IACrB,IAAIW,gBAAgB;IACpB,KAAK,MAAM,EAAEC,GAAG,EAAEL,UAAUM,YAAY,EAAEC,OAAO,EAAE,IAAI3B,QAAS;QAC9DoB,SAASjB,IAAI,IAAIuB;QACjBF,iBAAiBG,oBAAAA,qBAAAA,UAAW;QAC5B,KAAK,MAAMC,OAAOC,OAAOC,IAAI,CAACL,IAAIM,IAAI,EAAG;YACvC,IAAI,CAACT,KAAKU,GAAG,CAACJ,MAAM;gBAClBN,KAAKW,GAAG,CAACL;gBACTP,WAAWlB,IAAI,CAACyB;YAClB;QACF;QACAT,KAAKhB,IAAI,CAACsB;IACZ;IAEA,OAAO;QAAEN;QAAMC;QAAUC;QAAYG;IAAc;AACrD"}

package/dist/esm/store/builder.d.ts

@@ -1,11 +1,13 @@
 import type { Config } from '../config/index.js';
 import type { EmbedChangeKind } from './embed-scope.js';
 import type { FeatureToggle } from './feature-scope.js';
+import type { Stages } from './stages.js';
 import type { Connection, ReconcileDialect } from './types.js';
 export interface Builder {
     build(forcedPaths?: ReadonlySet<string>): Promise<{
         parsed: number;
         warnings: string[];
+        stages: Stages;
     }>;
     invalidate(kind: EmbedChangeKind): Promise<{
         parsed: number;

package/dist/esm/store/builder.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[] }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["embed","FEATURES","rank","listFiles","ParsePool","reparseFiles","NARROW_FEATURE_TABLE","reconcile","getColumns","withTransaction","nullEmbedVectors","conn","dialect","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","prepare","existingPaths","Set","all","map","r","path","files","filter","f","has","relPath","length","docs","undefined","paths","d","delta","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","name","feature","find","Error","turnedOn","docsForFeature","afterReconcile","nullRank","columns","rerunRank","invalidateFeatureToggles","toggles","byName","Map","linksToggle","get","rankToggle","createBuilder","build","forcedPaths","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":"AACA,SAASA,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,IAAI,QAAQ,sBAAsB;AAE3C,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,YAAY,QAAQ,qBAAqB;AAGlD,SAASC,oBAAoB,QAAQ,qBAAqB;AAC1D,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,UAAU,QAAQ,cAAc;AACzC,SAASC,eAAe,QAAQ,mBAAmB;AAsBnD,6FAA6F;AAC7F,wEAAwE;AACxE,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;IACzE,MAAMH,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,sDAAsDD,QAAQE,SAAS;IACnH,OAAO;QAAEC,QAAQ;IAAE;AACrB;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,eAAeC,kBAAkBL,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe;IACzH,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,gBAAgB,IAAIC,IAAI,AAAC,CAAA,MAAMH,aAAaI,GAAG,EAAC,EAAGC,GAAG,CAAC,CAACC,IAAM,AAACA,EAAuBC,IAAI;IAChG,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAMC,QAAQzB,UAAUc,KAAKC,SAASW,MAAM,CAAC,CAACC,IAAMA,EAAE9B,KAAK,IAAIsB,cAAcS,GAAG,CAACD,EAAEE,OAAO;IAC1F,IAAIJ,MAAMK,MAAM,KAAK,GAAG,OAAO;QAAElB,QAAQ;IAAE;IAE3C,MAAM,EAAEmB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAAC5B;KAAM,EAAEiB,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACtF,MAAMiB,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IAChF,MAAMhC,gBACJE,MACA;YACQX,eACAA;QADN,QAAMA,gBAAAA,MAAM0C,MAAM,cAAZ1C,oCAAAA,mBAAAA,OAAeW,MAAMyB,OAAOE;QAClC,QAAMtC,eAAAA,MAAM2C,KAAK,cAAX3C,mCAAAA,kBAAAA,OACJW,MACAuB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;gBAAEV,MAAMU,EAAEL,OAAO;gBAAEY,WAAWP,EAAEO,SAAS,CAAC5C,KAAK;YAAC,CAAA,IACjEsC;IAEJ,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,eAAeY,uBAAuBlC,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAE2B,MAAqB;IACrJ,MAAMC,QAAQzC,oBAAoB,CAACwC,OAAOE,IAAI,CAAC;IAC/C,MAAMC,UAAUhD,SAASiD,IAAI,CAAC,CAACpB,IAAMA,EAAEkB,IAAI,KAAKF,OAAOE,IAAI;IAC3D,IAAI,CAACC,SAAS,MAAM,IAAIE,MAAM,CAAC,qDAAqD,EAAEL,OAAOE,IAAI,CAAC,CAAC,CAAC;IAEpG,IAAI,CAACF,OAAOM,QAAQ,EAAE;QACpB,MAAM3C,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO,GAAGnC,QAAQE,SAAS;QACtF,OAAO;YAAEC,QAAQ;QAAE;IACrB;IAEA,MAAMa,QAAQzB,UAAUc,KAAKC;IAC7B,MAAM,EAAEgB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAACqB;KAAQ,EAAEhC,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACxF,MAAMkC,iBAAiBnB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;YAAEV,MAAMU,EAAEL,OAAO;YAAEY,WAAWP,EAAEO,SAAS,CAACK,QAAQD,IAAI,CAAC;QAAC,CAAA;IAChG,MAAMZ,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,gGAAgG;IAChG,8FAA8F;IAC9F,8CAA8C;IAC9C,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAOJ;QAAOK,UAAU,EAAE;IAAC;IACnF,MAAMhC,gBACJE,MACA;YAEQsC,gBACAA;QAFN,MAAMtC,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO;QACtC,QAAME,iBAAAA,QAAQN,KAAK,cAAbM,qCAAAA,oBAAAA,SAAgBtC,MAAM0C,gBAAgBf;QAC5C,QAAMW,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyBtC,MAAM2B;IACvC,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,eAAesB,SAAS5C,IAAgB,EAAEC,OAAyB;IACjE,MAAM4C,UAAU,MAAMhD,WAAWG;IACjC,IAAI,CAAC6C,QAAQzB,GAAG,CAAC,UAAU;IAC3B,MAAMtB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,0CAA0CD,QAAQE,SAAS;AACzG;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,eAAe2C,UAAU9C,IAAgB,EAAEC,OAAyB;IAClE,MAAM0B,QAAwB;QAAEV,OAAO,EAAE;QAAEW,UAAU,EAAE;QAAEC,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IACjF,MAAMhC,gBACJE,MACA;YACQT;QAAN,QAAMA,uBAAAA,KAAKoD,cAAc,cAAnBpD,2CAAAA,0BAAAA,MAAsBS,MAAM2B;IACpC,GACA1B,QAAQE,SAAS;AAErB;AAEA,eAAe4C,yBAAyB/C,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAEwC,OAAwB;IAC1J,MAAMC,SAAS,IAAIC,IAAIF,QAAQlC,GAAG,CAAC,CAACqB,SAAW;YAACA,OAAOE,IAAI;YAAEF;SAAO;IACpE,IAAI/B,SAAS;IAEb,4FAA4F;IAC5F,mFAAmF;IACnF,MAAM+C,cAAcF,OAAOG,GAAG,CAAC;IAC/B,IAAID,aAAa;QACf/C,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2C,YAAW,EAAG/C,MAAM;QAC/F,4FAA4F;QAC5F,0FAA0F;QAC1F,IAAI,CAAC+C,YAAYV,QAAQ,EAAE,MAAMG,SAAS5C,MAAMC;IAClD;IAEA,wFAAwF;IACxF,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMoD,aAAaJ,OAAOG,GAAG,CAAC;IAC9B,IAAIC,YAAY;QACd,IAAIA,WAAWZ,QAAQ,EAAE,MAAMK,UAAU9C,MAAMC;aAC1C,MAAM2C,SAAS5C,MAAMC;IAC5B;IAEA,KAAK,MAAMkC,UAAUa,QAAS;QAC5B,IAAIb,OAAOE,IAAI,KAAK,WAAWF,OAAOE,IAAI,KAAK,QAAQ;QACvDjC,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2B,OAAM,EAAG/B,MAAM;IAC5F;IACA,OAAO;QAAEA;IAAO;AAClB;AAEA,4FAA4F;AAC5F,6DAA6D;AAC7D,OAAO,SAASkD,cAActD,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB;IACrG,MAAMO,OAAO,IAAIf;IACjB,OAAO;QACL8D,OAAO,CAACC,cAAgB5D,UAAUI,MAAMM,KAAKC,SAASN,SAASO,MAAMgD;QACrEC,YAAY,CAACC,OAAUA,SAAS,UAAU3D,iBAAiBC,MAAMC,WAAWI,kBAAkBL,MAAMM,KAAKC,SAASN,SAASO;QAC3HmD,oBAAoB,CAACX,UAAYD,yBAAyB/C,MAAMM,KAAKC,SAASN,SAASO,MAAMwC;QAC7FY,OAAO,IAAMpD,KAAKoD,KAAK;QACvB,IAAIC;YACF,OAAOrD,KAAKqD,YAAY;QAC1B;IACF;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/builder.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { rank } from '../features/rank.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { listFiles } from '../scan/index.ts';\nimport { ParsePool } from '../scan/pool.ts';\nimport { reparseFiles } from '../scan/reparse.ts';\nimport type { EmbedChangeKind } from './embed-scope.ts';\nimport type { FeatureToggle } from './feature-scope.ts';\nimport { NARROW_FEATURE_TABLE } from './feature-scope.ts';\nimport { reconcile } from './reconcile.ts';\nimport { getColumns } from './shared.ts';\nimport type { Stages } from './stages.ts';\nimport { withTransaction } from './transaction.ts';\nimport type { Connection, ReconcileDialect } from './types.ts';\n\n// Owns bringing a store's index current: the write half `Store` (types.ts) deliberately does not\n// carry. A one-shot open calls build() once; a watcher calls it repeatedly on the same instance.\nexport interface Builder {\n  // forcedPaths (open.ts's preset-only narrow rebuild) applies to this call alone, not future ones.\n  build(forcedPaths?: ReadonlySet<string>): Promise<{ parsed: number; warnings: string[]; stages: Stages }>;\n  // Narrow embed invalidation (open.ts, embed-scope.ts's classifyEmbedChange): 'model' nulls every\n  // vector/scale in place; 'chunk' rebuilds every embedding row. Neither touches another table.\n  invalidate(kind: EmbedChangeKind): Promise<{ parsed: number }>;\n  // Narrow feature-toggle invalidation (open.ts, feature-scope.ts's classifyFeatureToggles): a\n  // toggle drops (or, turning back on, fully re-derives) that feature's own table; rank instead\n  // nulls or recomputes its `_rank` column, since rows written while a feature was off are\n  // untrustworthy (reconcile.ts's activeFeatures skips a disabled feature's hooks entirely).\n  invalidateFeatures(toggles: FeatureToggle[]): Promise<{ parsed: number }>;\n  // Releases the parse worker pool, if this lifetime ever created one; never the connection.\n  close(): Promise<void>;\n  // Pools this lifetime has constructed, so reuse is observable rather than inferred.\n  readonly poolsCreated: number;\n}\n\n// 'model': the provider or model name moved but chunk boundaries did not, so only the vector\n// values are stale. No reparse, and no table but embeddings is touched.\nasync function nullEmbedVectors(conn: Connection, dialect: ReconcileDialect): Promise<{ parsed: number }> {\n  await withTransaction(conn, () => conn.exec('UPDATE embeddings SET vector = NULL, scale = NULL'), dialect.beginMode());\n  return { parsed: 0 };\n}\n\n// 'chunk': chunkTokens or the chunk version moved, so chunk boundaries genuinely changed. Rebuilds\n// every embedding row through the embed feature's own remove/store, touching no other table.\nasync function rebuildEmbeddings(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool): Promise<{ parsed: number }> {\n  const existingStmt = await conn.prepare('SELECT \"path\" FROM frontmatter');\n  const existingPaths = new Set((await existingStmt.all()).map((r) => (r as { path: string }).path));\n  // A file added since the last build has no row yet; leaving it for build() to insert avoids a\n  // primary-key collision between this INSERT and build()'s own insert for the same new file.\n  const files = listFiles(cfg, baseDir).filter((f) => f.embed && existingPaths.has(f.relPath));\n  if (files.length === 0) return { parsed: 0 };\n\n  const { docs } = await reparseFiles(files, [embed], cfg, new Set(), undefined, { pool });\n  const paths = docs.map((d) => d.relPath);\n  const delta: ReconcileDelta = { files, reparsed: paths, added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await embed.remove?.(conn, paths, delta);\n      await embed.store?.(\n        conn,\n        docs.map((d) => ({ path: d.relPath, extracted: d.extracted.embed })),\n        delta\n      );\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// One feature's own table, dropped and (turning on) fully re-derived across every indexed file.\n// Rows left over from before this feature was disabled are untrustworthy by construction (files\n// added, edited, or deleted got no remove/store for it), so this never diffs against them --\n// dropping every row and reparsing the whole tree is the only safe path back to consistent state.\nasync function invalidateFeatureTable(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggle: FeatureToggle): Promise<{ parsed: number }> {\n  const table = NARROW_FEATURE_TABLE[toggle.name];\n  const feature = FEATURES.find((f) => f.name === toggle.name);\n  if (!feature) throw new Error(`invalidateFeatureTable: no registered feature named \"${toggle.name}\"`);\n\n  if (!toggle.turnedOn) {\n    await withTransaction(conn, () => conn.exec(`DELETE FROM ${table}`), dialect.beginMode());\n    return { parsed: 0 };\n  }\n\n  const files = listFiles(cfg, baseDir);\n  const { docs } = await reparseFiles(files, [feature], cfg, new Set(), undefined, { pool });\n  const docsForFeature = docs.map((d) => ({ path: d.relPath, extracted: d.extracted[feature.name] }));\n  const paths = docs.map((d) => d.relPath);\n  // Every path counts as added, not just touched: the table was just dropped, so there is nothing\n  // stale to diff against, and links' own afterReconcile takes its cold-build (resolveAll) path\n  // only when added.length equals files.length.\n  const delta: ReconcileDelta = { files, reparsed: paths, added: paths, vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await conn.exec(`DELETE FROM ${table}`);\n      await feature.store?.(conn, docsForFeature, delta);\n      await feature.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n  return { parsed: docs.length };\n}\n\n// rank has no table of its own: frontmatter._rank is the only state it owns. Guarded on column\n// presence, since links can toggle off while rank has never been enabled for this tree.\nasync function nullRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const columns = await getColumns(conn);\n  if (!columns.has('_rank')) return;\n  await withTransaction(conn, () => conn.exec('UPDATE frontmatter SET \"_rank\" = NULL'), dialect.beginMode());\n}\n\n// Whole-tree and reparse-free: PageRank is computed from the links table, not from files, so\n// turning rank back on needs no reparseFiles pass at all.\nasync function rerunRank(conn: Connection, dialect: ReconcileDialect): Promise<void> {\n  const delta: ReconcileDelta = { files: [], reparsed: [], added: [], vanished: [] };\n  await withTransaction(\n    conn,\n    async () => {\n      await rank.afterReconcile?.(conn, delta);\n    },\n    dialect.beginMode()\n  );\n}\n\nasync function invalidateFeatureToggles(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect, pool: ParsePool, toggles: FeatureToggle[]): Promise<{ parsed: number }> {\n  const byName = new Map(toggles.map((toggle) => [toggle.name, toggle]));\n  let parsed = 0;\n\n  // links first, if present: rank's rerun below (whether from its own toggle or the co-change\n  // rankToggle carries) must read the links table after it is back in its new state.\n  const linksToggle = byName.get('links');\n  if (linksToggle) {\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, linksToggle)).parsed;\n    // Unconditional, per rank's dependency on links: nulling is a no-op (guarded) when rank was\n    // never enabled for this tree, so this is safe even when no feature:rank segment changed.\n    if (!linksToggle.turnedOn) await nullRank(conn, dialect);\n  }\n\n  // rank's effective on/off (featureEnabled's links dependency) only appears as a changed\n  // `feature:rank` segment when it actually flips -- whether that's rank's own config or a\n  // links toggle taking it along -- so honoring it here is complete on its own.\n  const rankToggle = byName.get('rank');\n  if (rankToggle) {\n    if (rankToggle.turnedOn) await rerunRank(conn, dialect);\n    else await nullRank(conn, dialect);\n  }\n\n  for (const toggle of toggles) {\n    if (toggle.name === 'links' || toggle.name === 'rank') continue;\n    parsed += (await invalidateFeatureTable(conn, cfg, baseDir, dialect, pool, toggle)).parsed;\n  }\n  return { parsed };\n}\n\n// The pool is created at most once, lazily, on whichever build() or invalidate() call first\n// needs it, and reused by every later call on this instance.\nexport function createBuilder(conn: Connection, cfg: Config, baseDir: string, dialect: ReconcileDialect): Builder {\n  const pool = new ParsePool();\n  return {\n    build: (forcedPaths) => reconcile(conn, cfg, baseDir, dialect, pool, forcedPaths),\n    invalidate: (kind) => (kind === 'model' ? nullEmbedVectors(conn, dialect) : rebuildEmbeddings(conn, cfg, baseDir, dialect, pool)),\n    invalidateFeatures: (toggles) => invalidateFeatureToggles(conn, cfg, baseDir, dialect, pool, toggles),\n    close: () => pool.close(),\n    get poolsCreated() {\n      return pool.poolsCreated;\n    },\n  };\n}\n"],"names":["embed","FEATURES","rank","listFiles","ParsePool","reparseFiles","NARROW_FEATURE_TABLE","reconcile","getColumns","withTransaction","nullEmbedVectors","conn","dialect","exec","beginMode","parsed","rebuildEmbeddings","cfg","baseDir","pool","existingStmt","prepare","existingPaths","Set","all","map","r","path","files","filter","f","has","relPath","length","docs","undefined","paths","d","delta","reparsed","added","vanished","remove","store","extracted","invalidateFeatureTable","toggle","table","name","feature","find","Error","turnedOn","docsForFeature","afterReconcile","nullRank","columns","rerunRank","invalidateFeatureToggles","toggles","byName","Map","linksToggle","get","rankToggle","createBuilder","build","forcedPaths","invalidate","kind","invalidateFeatures","close","poolsCreated"],"mappings":"AACA,SAASA,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,IAAI,QAAQ,sBAAsB;AAE3C,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,SAAS,QAAQ,kBAAkB;AAC5C,SAASC,YAAY,QAAQ,qBAAqB;AAGlD,SAASC,oBAAoB,QAAQ,qBAAqB;AAC1D,SAASC,SAAS,QAAQ,iBAAiB;AAC3C,SAASC,UAAU,QAAQ,cAAc;AAEzC,SAASC,eAAe,QAAQ,mBAAmB;AAsBnD,6FAA6F;AAC7F,wEAAwE;AACxE,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAyB;IACzE,MAAMH,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,sDAAsDD,QAAQE,SAAS;IACnH,OAAO;QAAEC,QAAQ;IAAE;AACrB;AAEA,mGAAmG;AACnG,6FAA6F;AAC7F,eAAeC,kBAAkBL,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe;IACzH,MAAMC,eAAe,MAAMT,KAAKU,OAAO,CAAC;IACxC,MAAMC,gBAAgB,IAAIC,IAAI,AAAC,CAAA,MAAMH,aAAaI,GAAG,EAAC,EAAGC,GAAG,CAAC,CAACC,IAAM,AAACA,EAAuBC,IAAI;IAChG,8FAA8F;IAC9F,4FAA4F;IAC5F,MAAMC,QAAQzB,UAAUc,KAAKC,SAASW,MAAM,CAAC,CAACC,IAAMA,EAAE9B,KAAK,IAAIsB,cAAcS,GAAG,CAACD,EAAEE,OAAO;IAC1F,IAAIJ,MAAMK,MAAM,KAAK,GAAG,OAAO;QAAElB,QAAQ;IAAE;IAE3C,MAAM,EAAEmB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAAC5B;KAAM,EAAEiB,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACtF,MAAMiB,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IAChF,MAAMhC,gBACJE,MACA;YACQX,eACAA;QADN,QAAMA,gBAAAA,MAAM0C,MAAM,cAAZ1C,oCAAAA,mBAAAA,OAAeW,MAAMyB,OAAOE;QAClC,QAAMtC,eAAAA,MAAM2C,KAAK,cAAX3C,mCAAAA,kBAAAA,OACJW,MACAuB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;gBAAEV,MAAMU,EAAEL,OAAO;gBAAEY,WAAWP,EAAEO,SAAS,CAAC5C,KAAK;YAAC,CAAA,IACjEsC;IAEJ,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,6FAA6F;AAC7F,kGAAkG;AAClG,eAAeY,uBAAuBlC,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAE2B,MAAqB;IACrJ,MAAMC,QAAQzC,oBAAoB,CAACwC,OAAOE,IAAI,CAAC;IAC/C,MAAMC,UAAUhD,SAASiD,IAAI,CAAC,CAACpB,IAAMA,EAAEkB,IAAI,KAAKF,OAAOE,IAAI;IAC3D,IAAI,CAACC,SAAS,MAAM,IAAIE,MAAM,CAAC,qDAAqD,EAAEL,OAAOE,IAAI,CAAC,CAAC,CAAC;IAEpG,IAAI,CAACF,OAAOM,QAAQ,EAAE;QACpB,MAAM3C,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO,GAAGnC,QAAQE,SAAS;QACtF,OAAO;YAAEC,QAAQ;QAAE;IACrB;IAEA,MAAMa,QAAQzB,UAAUc,KAAKC;IAC7B,MAAM,EAAEgB,IAAI,EAAE,GAAG,MAAM7B,aAAauB,OAAO;QAACqB;KAAQ,EAAEhC,KAAK,IAAIM,OAAOY,WAAW;QAAEhB;IAAK;IACxF,MAAMkC,iBAAiBnB,KAAKT,GAAG,CAAC,CAACY,IAAO,CAAA;YAAEV,MAAMU,EAAEL,OAAO;YAAEY,WAAWP,EAAEO,SAAS,CAACK,QAAQD,IAAI,CAAC;QAAC,CAAA;IAChG,MAAMZ,QAAQF,KAAKT,GAAG,CAAC,CAACY,IAAMA,EAAEL,OAAO;IACvC,gGAAgG;IAChG,8FAA8F;IAC9F,8CAA8C;IAC9C,MAAMM,QAAwB;QAAEV;QAAOW,UAAUH;QAAOI,OAAOJ;QAAOK,UAAU,EAAE;IAAC;IACnF,MAAMhC,gBACJE,MACA;YAEQsC,gBACAA;QAFN,MAAMtC,KAAKE,IAAI,CAAC,CAAC,YAAY,EAAEkC,OAAO;QACtC,QAAME,iBAAAA,QAAQN,KAAK,cAAbM,qCAAAA,oBAAAA,SAAgBtC,MAAM0C,gBAAgBf;QAC5C,QAAMW,0BAAAA,QAAQK,cAAc,cAAtBL,8CAAAA,6BAAAA,SAAyBtC,MAAM2B;IACvC,GACA1B,QAAQE,SAAS;IAEnB,OAAO;QAAEC,QAAQmB,KAAKD,MAAM;IAAC;AAC/B;AAEA,+FAA+F;AAC/F,wFAAwF;AACxF,eAAesB,SAAS5C,IAAgB,EAAEC,OAAyB;IACjE,MAAM4C,UAAU,MAAMhD,WAAWG;IACjC,IAAI,CAAC6C,QAAQzB,GAAG,CAAC,UAAU;IAC3B,MAAMtB,gBAAgBE,MAAM,IAAMA,KAAKE,IAAI,CAAC,0CAA0CD,QAAQE,SAAS;AACzG;AAEA,6FAA6F;AAC7F,0DAA0D;AAC1D,eAAe2C,UAAU9C,IAAgB,EAAEC,OAAyB;IAClE,MAAM0B,QAAwB;QAAEV,OAAO,EAAE;QAAEW,UAAU,EAAE;QAAEC,OAAO,EAAE;QAAEC,UAAU,EAAE;IAAC;IACjF,MAAMhC,gBACJE,MACA;YACQT;QAAN,QAAMA,uBAAAA,KAAKoD,cAAc,cAAnBpD,2CAAAA,0BAAAA,MAAsBS,MAAM2B;IACpC,GACA1B,QAAQE,SAAS;AAErB;AAEA,eAAe4C,yBAAyB/C,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB,EAAEO,IAAe,EAAEwC,OAAwB;IAC1J,MAAMC,SAAS,IAAIC,IAAIF,QAAQlC,GAAG,CAAC,CAACqB,SAAW;YAACA,OAAOE,IAAI;YAAEF;SAAO;IACpE,IAAI/B,SAAS;IAEb,4FAA4F;IAC5F,mFAAmF;IACnF,MAAM+C,cAAcF,OAAOG,GAAG,CAAC;IAC/B,IAAID,aAAa;QACf/C,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2C,YAAW,EAAG/C,MAAM;QAC/F,4FAA4F;QAC5F,0FAA0F;QAC1F,IAAI,CAAC+C,YAAYV,QAAQ,EAAE,MAAMG,SAAS5C,MAAMC;IAClD;IAEA,wFAAwF;IACxF,yFAAyF;IACzF,8EAA8E;IAC9E,MAAMoD,aAAaJ,OAAOG,GAAG,CAAC;IAC9B,IAAIC,YAAY;QACd,IAAIA,WAAWZ,QAAQ,EAAE,MAAMK,UAAU9C,MAAMC;aAC1C,MAAM2C,SAAS5C,MAAMC;IAC5B;IAEA,KAAK,MAAMkC,UAAUa,QAAS;QAC5B,IAAIb,OAAOE,IAAI,KAAK,WAAWF,OAAOE,IAAI,KAAK,QAAQ;QACvDjC,UAAU,AAAC,CAAA,MAAM8B,uBAAuBlC,MAAMM,KAAKC,SAASN,SAASO,MAAM2B,OAAM,EAAG/B,MAAM;IAC5F;IACA,OAAO;QAAEA;IAAO;AAClB;AAEA,4FAA4F;AAC5F,6DAA6D;AAC7D,OAAO,SAASkD,cAActD,IAAgB,EAAEM,GAAW,EAAEC,OAAe,EAAEN,OAAyB;IACrG,MAAMO,OAAO,IAAIf;IACjB,OAAO;QACL8D,OAAO,CAACC,cAAgB5D,UAAUI,MAAMM,KAAKC,SAASN,SAASO,MAAMgD;QACrEC,YAAY,CAACC,OAAUA,SAAS,UAAU3D,iBAAiBC,MAAMC,WAAWI,kBAAkBL,MAAMM,KAAKC,SAASN,SAASO;QAC3HmD,oBAAoB,CAACX,UAAYD,yBAAyB/C,MAAMM,KAAKC,SAASN,SAASO,MAAMwC;QAC7FY,OAAO,IAAMpD,KAAKoD,KAAK;QACvB,IAAIC;YACF,OAAOrD,KAAKqD,YAAY;QAC1B;IACF;AACF"}

package/dist/esm/store/duckdb/connection.d.ts

@@ -2,4 +2,7 @@ import type { DuckDBConnection } from '@duckdb/node-api';
 import type { Connection } from '../types.js';
 export declare function storeValueToJs(value: unknown): unknown;
 export declare function storeRowToJs(row: Record<string, unknown>): Record<string, unknown>;
-export declare function createConnection(duckdb: DuckDBConnection): Connection;
+export interface DuckdbConnection extends Connection {
+    readonly duckdb: DuckDBConnection;
+}
+export declare function createConnection(duckdb: DuckDBConnection): DuckdbConnection;

package/dist/esm/store/duckdb/connection.js

@@ -59,6 +59,7 @@ let DuckdbStatement = class DuckdbStatement {
 };
 export function createConnection(duckdb) {
     const conn = {
+        duckdb,
         async exec (sql) {
             await duckdb.run(sql);
         },

package/dist/esm/store/duckdb/connection.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\nexport function createConnection(duckdb: DuckDBConnection): Connection {\n  const conn: Connection = {\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAEA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAmB;QACvB,MAAMC,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/connection.ts"],"sourcesContent":["import type { DuckDBConnection, DuckDBPreparedStatement, DuckDBValue } from '@duckdb/node-api';\nimport { withTransaction } from '../transaction.ts';\nimport type { Connection, RunResult, Statement } from '../types.ts';\nimport { rewriteBatch } from './batch.ts';\n\n// getRowObjectsJS returns INT64 columns as BigInt regardless of magnitude, while sqlite's small\n// ints are numbers and consumers assume number; in-range values convert here, out-of-range stays BigInt.\nconst MAX_SAFE = BigInt(Number.MAX_SAFE_INTEGER);\nexport function storeValueToJs(value: unknown): unknown {\n  return typeof value === 'bigint' && value >= -MAX_SAFE && value <= MAX_SAFE ? Number(value) : value;\n}\nexport function storeRowToJs(row: Record<string, unknown>): Record<string, unknown> {\n  const out: Record<string, unknown> = {};\n  for (const [key, value] of Object.entries(row)) out[key] = storeValueToJs(value);\n  return out;\n}\n\n// @duckdb/node-api only destroys a prepared statement when its connection closes, but a connection can stay open for a whole `sense watch` session.\n// Statement has no dispose (callers reuse one instance across several calls), so this FinalizationRegistry reclaims it once nothing references the wrapper.\nconst preparedFinalizer = new FinalizationRegistry<DuckDBPreparedStatement>((prepared) => {\n  prepared.destroySync();\n});\n\n// Wraps one already-prepared DuckDB statement: prepare() is the only async step, so run/get/all rebind and\n// re-execute the same native statement, matching how callers like graph/traverse.ts's ring loop reuse one Statement across several calls.\nclass DuckdbStatement implements Statement {\n  private prepared: DuckDBPreparedStatement;\n\n  constructor(prepared: DuckDBPreparedStatement) {\n    this.prepared = prepared;\n    preparedFinalizer.register(this, prepared);\n  }\n\n  async run(...params: unknown[]): Promise<RunResult> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const result = await this.prepared.run();\n    return { changes: result.rowsChanged, lastInsertRowid: 0 };\n  }\n\n  async get(...params: unknown[]): Promise<unknown> {\n    const rows = await this.all(...params);\n    return rows[0];\n  }\n\n  async all(...params: unknown[]): Promise<unknown[]> {\n    if (params.length > 0) this.prepared.bind(params as DuckDBValue[]);\n    const reader = await this.prepared.runAndReadAll();\n    return (reader.getRowObjectsJS() as Array<Record<string, unknown>>).map(storeRowToJs);\n  }\n\n  async *iterate(...params: unknown[]): AsyncIterable<unknown> {\n    // Never called on the portable Connection/Statement surface today (only Store.raw streams);\n    // materializing keeps this a correct, if eager, AsyncIterable.\n    yield* await this.all(...params);\n  }\n\n  columns(): Array<{ name: string }> {\n    const out: Array<{ name: string }> = [];\n    for (let i = 0; i < this.prepared.columnCount; i++) out.push({ name: this.prepared.columnName(i) });\n    return out;\n  }\n\n  setReadBigInts(_enabled: boolean): void {\n    // No-op: in-range ints are already numbers here (storeRowToJs converts what getRowObjectsJS\n    // hands back as BigInt), so the node:sqlite throw-at-step-time problem does not exist.\n  }\n}\n\n// Adds the native DuckDBConnection to the portable Connection, so duckdb's own dialect code\n// (reconcile.ts's insertNew) can reach createAppender(); sqlite/turso have no such member.\nexport interface DuckdbConnection extends Connection {\n  readonly duckdb: DuckDBConnection;\n}\n\nexport function createConnection(duckdb: DuckDBConnection): DuckdbConnection {\n  const conn: DuckdbConnection = {\n    duckdb,\n    async exec(sql: string): Promise<void> {\n      await duckdb.run(sql);\n    },\n    async prepare(sql: string): Promise<Statement> {\n      return new DuckdbStatement(await duckdb.prepare(sql));\n    },\n    // One crossing regardless of row count: rewriteBatch folds recognized shapes into a multi-row statement (batch.ts),\n    // else falls back to a bind-and-run loop, one call either way. Joins the caller's transaction when there is one, else opens its own.\n    async runBatch(sql: string, paramRows: unknown[][]): Promise<void> {\n      if (paramRows.length === 0) return;\n      await withTransaction(conn, async () => {\n        const rewritten = rewriteBatch(sql, paramRows.length);\n        if (rewritten) {\n          const stmt = await duckdb.prepare(rewritten.sql);\n          try {\n            stmt.bind(paramRows.flat() as DuckDBValue[]);\n            await stmt.run();\n          } finally {\n            stmt.destroySync();\n          }\n          return;\n        }\n        const stmt = await duckdb.prepare(sql);\n        try {\n          for (const row of paramRows) {\n            stmt.bind(row as DuckDBValue[]);\n            await stmt.run();\n          }\n        } finally {\n          stmt.destroySync();\n        }\n      });\n    },\n  };\n  return conn;\n}\n"],"names":["withTransaction","rewriteBatch","MAX_SAFE","BigInt","Number","MAX_SAFE_INTEGER","storeValueToJs","value","storeRowToJs","row","out","key","Object","entries","preparedFinalizer","FinalizationRegistry","prepared","destroySync","DuckdbStatement","run","params","length","bind","result","changes","rowsChanged","lastInsertRowid","get","rows","all","reader","runAndReadAll","getRowObjectsJS","map","iterate","columns","i","columnCount","push","name","columnName","setReadBigInts","_enabled","register","createConnection","duckdb","conn","exec","sql","prepare","runBatch","paramRows","rewritten","stmt","flat"],"mappings":"AACA,SAASA,eAAe,QAAQ,oBAAoB;AAEpD,SAASC,YAAY,QAAQ,aAAa;AAE1C,gGAAgG;AAChG,yGAAyG;AACzG,MAAMC,WAAWC,OAAOC,OAAOC,gBAAgB;AAC/C,OAAO,SAASC,eAAeC,KAAc;IAC3C,OAAO,OAAOA,UAAU,YAAYA,SAAS,CAACL,YAAYK,SAASL,WAAWE,OAAOG,SAASA;AAChG;AACA,OAAO,SAASC,aAAaC,GAA4B;IACvD,MAAMC,MAA+B,CAAC;IACtC,KAAK,MAAM,CAACC,KAAKJ,MAAM,IAAIK,OAAOC,OAAO,CAACJ,KAAMC,GAAG,CAACC,IAAI,GAAGL,eAAeC;IAC1E,OAAOG;AACT;AAEA,oJAAoJ;AACpJ,4JAA4J;AAC5J,MAAMI,oBAAoB,IAAIC,qBAA8C,CAACC;IAC3EA,SAASC,WAAW;AACtB;AAEA,2GAA2G;AAC3G,0IAA0I;AAC1I,IAAA,AAAMC,kBAAN,MAAMA;IAQJ,MAAMC,IAAI,GAAGC,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMG,SAAS,MAAM,IAAI,CAACP,QAAQ,CAACG,GAAG;QACtC,OAAO;YAAEK,SAASD,OAAOE,WAAW;YAAEC,iBAAiB;QAAE;IAC3D;IAEA,MAAMC,IAAI,GAAGP,MAAiB,EAAoB;QAChD,MAAMQ,OAAO,MAAM,IAAI,CAACC,GAAG,IAAIT;QAC/B,OAAOQ,IAAI,CAAC,EAAE;IAChB;IAEA,MAAMC,IAAI,GAAGT,MAAiB,EAAsB;QAClD,IAAIA,OAAOC,MAAM,GAAG,GAAG,IAAI,CAACL,QAAQ,CAACM,IAAI,CAACF;QAC1C,MAAMU,SAAS,MAAM,IAAI,CAACd,QAAQ,CAACe,aAAa;QAChD,OAAO,AAACD,OAAOE,eAAe,GAAsCC,GAAG,CAACzB;IAC1E;IAEA,OAAO0B,QAAQ,GAAGd,MAAiB,EAA0B;QAC3D,4FAA4F;QAC5F,+DAA+D;QAC/D,OAAO,MAAM,IAAI,CAACS,GAAG,IAAIT;IAC3B;IAEAe,UAAmC;QACjC,MAAMzB,MAA+B,EAAE;QACvC,IAAK,IAAI0B,IAAI,GAAGA,IAAI,IAAI,CAACpB,QAAQ,CAACqB,WAAW,EAAED,IAAK1B,IAAI4B,IAAI,CAAC;YAAEC,MAAM,IAAI,CAACvB,QAAQ,CAACwB,UAAU,CAACJ;QAAG;QACjG,OAAO1B;IACT;IAEA+B,eAAeC,QAAiB,EAAQ;IACtC,4FAA4F;IAC5F,uFAAuF;IACzF;IArCA,YAAY1B,QAAiC,CAAE;QAC7C,IAAI,CAACA,QAAQ,GAAGA;QAChBF,kBAAkB6B,QAAQ,CAAC,IAAI,EAAE3B;IACnC;AAmCF;AAQA,OAAO,SAAS4B,iBAAiBC,MAAwB;IACvD,MAAMC,OAAyB;QAC7BD;QACA,MAAME,MAAKC,GAAW;YACpB,MAAMH,OAAO1B,GAAG,CAAC6B;QACnB;QACA,MAAMC,SAAQD,GAAW;YACvB,OAAO,IAAI9B,gBAAgB,MAAM2B,OAAOI,OAAO,CAACD;QAClD;QACA,oHAAoH;QACpH,qIAAqI;QACrI,MAAME,UAASF,GAAW,EAAEG,SAAsB;YAChD,IAAIA,UAAU9B,MAAM,KAAK,GAAG;YAC5B,MAAMrB,gBAAgB8C,MAAM;gBAC1B,MAAMM,YAAYnD,aAAa+C,KAAKG,UAAU9B,MAAM;gBACpD,IAAI+B,WAAW;oBACb,MAAMC,OAAO,MAAMR,OAAOI,OAAO,CAACG,UAAUJ,GAAG;oBAC/C,IAAI;wBACFK,KAAK/B,IAAI,CAAC6B,UAAUG,IAAI;wBACxB,MAAMD,KAAKlC,GAAG;oBAChB,SAAU;wBACRkC,KAAKpC,WAAW;oBAClB;oBACA;gBACF;gBACA,MAAMoC,OAAO,MAAMR,OAAOI,OAAO,CAACD;gBAClC,IAAI;oBACF,KAAK,MAAMvC,OAAO0C,UAAW;wBAC3BE,KAAK/B,IAAI,CAACb;wBACV,MAAM4C,KAAKlC,GAAG;oBAChB;gBACF,SAAU;oBACRkC,KAAKpC,WAAW;gBAClB;YACF;QACF;IACF;IACA,OAAO6B;AACT"}

package/dist/esm/store/duckdb/reconcile.js

@@ -1,6 +1,8 @@
 import { SenseError } from '../../errors.js';
+import { CORE_FRONTMATTER_COLUMNS } from '../reconcile.js';
 import { quoteIdent } from '../shared.js';
 import { markContentStale } from './lexical.js';
+import { duckdbApi } from './native.js';
 // This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in
 // store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here
 // unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter
@@ -37,6 +39,36 @@ async function addColumns(conn, names) {
     if (names.length === 0) return;
     await conn.exec(names.map((name)=>`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));
 }
+// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so
+// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. "_rank") that still exist on the table, and get appendDefault().
+async function insertNew(conn, table, columns, rows) {
+    if (rows.length === 0) return;
+    const { variantValue } = await duckdbApi();
+    const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);
+    const physicalColumns = (await infoStmt.all()).map((c)=>c.name);
+    const rowIndexOf = new Map(columns.map((name, i)=>[
+            name,
+            i
+        ]));
+    const native = conn.duckdb;
+    const appender = await native.createAppender(table);
+    try {
+        for (const row of rows){
+            for (const name of physicalColumns){
+                const idx = rowIndexOf.get(name);
+                const value = idx === undefined ? undefined : row[idx];
+                if (idx === undefined) appender.appendDefault();
+                else if (value === null || value === undefined) appender.appendNull();
+                else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value);
+                else appender.appendVariant(variantValue(value));
+            }
+            appender.endRow();
+        }
+        appender.flushSync();
+    } finally{
+        appender.closeSync();
+    }
+}
 export const duckdbDialect = {
     beginMode: ()=>'BEGIN',
     checkColumnLimit (count) {
@@ -45,5 +77,6 @@ export const duckdbDialect = {
         }
     },
     addColumns,
-    reconcileContent
+    reconcileContent,
+    insertNew
 };

package/dist/esm/store/duckdb/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport { markContentStale } from './lexical.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n};\n"],"names":["SenseError","quoteIdent","markContentStale","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","addColumns","names","exec","name","join","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AAAA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,gBAAgB,QAAQ,eAAe;AAEhD,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;IACtE,6FAA6F;IAC7F,yCAAyC;IACzCF,iBAAiBW;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeQ,WAAWR,IAAgB,EAAES,KAAe;IACzD,IAAIA,MAAML,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKU,IAAI,CAACD,MAAMH,GAAG,CAAC,CAACK,OAAS,CAAC,mCAAmC,EAAEvB,WAAWuB,MAAM,QAAQ,CAAC,EAAEC,IAAI,CAAC;AAC7G;AAEA,OAAO,MAAMC,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQlB,yBAAyB;YACnC,MAAM,IAAIX,WAAW,gBAAgB,CAAC,uBAAuB,EAAE6B,MAAM,8CAA8C,EAAElB,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAU;IACAT;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/store/duckdb/reconcile.ts"],"sourcesContent":["import type { DuckDBValue } from '@duckdb/node-api';\nimport { SenseError } from '../../errors.ts';\nimport type { ReconcileDelta } from '../../features/types.ts';\nimport type { ParsedDoc } from '../../scan/index.ts';\nimport { CORE_FRONTMATTER_COLUMNS } from '../reconcile.ts';\nimport { quoteIdent } from '../shared.ts';\nimport type { Connection, ReconcileDialect } from '../types.ts';\nimport type { DuckdbConnection } from './connection.ts';\nimport { markContentStale } from './lexical.ts';\nimport { duckdbApi } from './native.ts';\n\n// This store's dialect (types.ts's ReconcileDialect) for the shared orchestration in\n// store/reconcile.ts. `content` is a plain table, not FTS-virtual, so rows are maintained here\n// unconditionally; DuckDB's ALTER TABLE needs a declared type, so every dynamic frontmatter\n// column is VARIANT (holds mapValue()'s mixed JS types for one key across files).\n\n// No rowid coupling needed (unlike sqlite's content, which links to frontmatter's rowid):\n// `path` is content's own primary key, so this is a plain per-doc row.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (\"path\", title, summary, text) VALUES (?, ?, ?, ?)`;\n\nfunction contentRow(doc: ParsedDoc): unknown[] {\n  return [doc.relPath, doc.search.title, doc.search.summary, doc.search.text];\n}\n\n// No compile-time column cap in DuckDB (unlike SQLite's SQLITE_MAX_COLUMN); kept as a sanity fence anyway\n// so a runaway frontmatter generator fails with a clear message instead of an unbounded ALTER TABLE loop.\nconst MAX_FRONTMATTER_COLUMNS = 10_000;\n\nasync function reconcileContent(conn: Connection, touched: string[], docs: ParsedDoc[], _delta: ReconcileDelta): Promise<void> {\n  if (touched.length > 0)\n    await conn.runBatch(\n      'DELETE FROM content WHERE \"path\" = ?',\n      touched.map((p) => [p])\n    );\n  if (docs.length > 0) await conn.runBatch(INSERT_CONTENT_SQL, docs.map(contentRow));\n  // content changed: the fts index is rebuilt lazily, on the next lexical query that needs it,\n  // not here (lexical.ts's FtsIndexState).\n  markContentStale(conn);\n}\n\n// DuckDB rejects more than one ALTER command per statement (\"Parser Error: Only one ALTER\n// command per statement is supported\", measured), so every name's clause joins into one string\n// and runs as a single exec() -- one column-add \"leg\" through the driver instead of `names.length`.\n// VARIANT is the only type that can hold the mixed bigint/number/string/null shapes mapValue()\n// produces for one key across files.\nasync function addColumns(conn: Connection, names: string[]): Promise<void> {\n  if (names.length === 0) return;\n  await conn.exec(names.map((name) => `ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(name)} VARIANT`).join('; '));\n}\n\n// Appender path for rows that cannot conflict (reconcile.ts's `added`); no ON CONFLICT support, so\n// alignment reads the table's own physical column order fresh -- `columns` omits feature-owned reserved columns (e.g. \"_rank\") that still exist on the table, and get appendDefault().\nasync function insertNew(conn: Connection, table: string, columns: string[], rows: unknown[][]): Promise<void> {\n  if (rows.length === 0) return;\n  const { variantValue } = await duckdbApi();\n  const infoStmt = await conn.prepare(`PRAGMA table_info(${quoteIdent(table)})`);\n  const physicalColumns = ((await infoStmt.all()) as Array<{ name: string }>).map((c) => c.name);\n  const rowIndexOf = new Map(columns.map((name, i) => [name, i]));\n\n  const native = (conn as DuckdbConnection).duckdb;\n  const appender = await native.createAppender(table);\n  try {\n    for (const row of rows) {\n      for (const name of physicalColumns) {\n        const idx = rowIndexOf.get(name);\n        const value = idx === undefined ? undefined : row[idx];\n        if (idx === undefined) appender.appendDefault();\n        else if (value === null || value === undefined) appender.appendNull();\n        else if (CORE_FRONTMATTER_COLUMNS.has(name)) appender.appendValue(value as DuckDBValue);\n        else appender.appendVariant(variantValue(value as DuckDBValue));\n      }\n      appender.endRow();\n    }\n    appender.flushSync();\n  } finally {\n    appender.closeSync();\n  }\n}\n\nexport const duckdbDialect: ReconcileDialect = {\n  beginMode: () => 'BEGIN',\n  checkColumnLimit(count) {\n    if (count > MAX_FRONTMATTER_COLUMNS) {\n      throw new SenseError('COLUMN_LIMIT', `frontmatter would need ${count} columns, crossing this store's sanity limit (${MAX_FRONTMATTER_COLUMNS}). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`);\n    }\n  },\n  addColumns,\n  reconcileContent,\n  insertNew,\n};\n"],"names":["SenseError","CORE_FRONTMATTER_COLUMNS","quoteIdent","markContentStale","duckdbApi","INSERT_CONTENT_SQL","contentRow","doc","relPath","search","title","summary","text","MAX_FRONTMATTER_COLUMNS","reconcileContent","conn","touched","docs","_delta","length","runBatch","map","p","addColumns","names","exec","name","join","insertNew","table","columns","rows","variantValue","infoStmt","prepare","physicalColumns","all","c","rowIndexOf","Map","i","native","duckdb","appender","createAppender","row","idx","get","value","undefined","appendDefault","appendNull","has","appendValue","appendVariant","endRow","flushSync","closeSync","duckdbDialect","beginMode","checkColumnLimit","count"],"mappings":"AACA,SAASA,UAAU,QAAQ,kBAAkB;AAG7C,SAASC,wBAAwB,QAAQ,kBAAkB;AAC3D,SAASC,UAAU,QAAQ,eAAe;AAG1C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,SAAS,QAAQ,cAAc;AAExC,qFAAqF;AACrF,+FAA+F;AAC/F,4FAA4F;AAC5F,kFAAkF;AAElF,0FAA0F;AAC1F,uEAAuE;AACvE,MAAMC,qBAAqB,CAAC,sEAAsE,CAAC;AAEnG,SAASC,WAAWC,GAAc;IAChC,OAAO;QAACA,IAAIC,OAAO;QAAED,IAAIE,MAAM,CAACC,KAAK;QAAEH,IAAIE,MAAM,CAACE,OAAO;QAAEJ,IAAIE,MAAM,CAACG,IAAI;KAAC;AAC7E;AAEA,0GAA0G;AAC1G,0GAA0G;AAC1G,MAAMC,0BAA0B;AAEhC,eAAeC,iBAAiBC,IAAgB,EAAEC,OAAiB,EAAEC,IAAiB,EAAEC,MAAsB;IAC5G,IAAIF,QAAQG,MAAM,GAAG,GACnB,MAAMJ,KAAKK,QAAQ,CACjB,wCACAJ,QAAQK,GAAG,CAAC,CAACC,IAAM;YAACA;SAAE;IAE1B,IAAIL,KAAKE,MAAM,GAAG,GAAG,MAAMJ,KAAKK,QAAQ,CAACf,oBAAoBY,KAAKI,GAAG,CAACf;IACtE,6FAA6F;IAC7F,yCAAyC;IACzCH,iBAAiBY;AACnB;AAEA,0FAA0F;AAC1F,+FAA+F;AAC/F,oGAAoG;AACpG,+FAA+F;AAC/F,qCAAqC;AACrC,eAAeQ,WAAWR,IAAgB,EAAES,KAAe;IACzD,IAAIA,MAAML,MAAM,KAAK,GAAG;IACxB,MAAMJ,KAAKU,IAAI,CAACD,MAAMH,GAAG,CAAC,CAACK,OAAS,CAAC,mCAAmC,EAAExB,WAAWwB,MAAM,QAAQ,CAAC,EAAEC,IAAI,CAAC;AAC7G;AAEA,mGAAmG;AACnG,uLAAuL;AACvL,eAAeC,UAAUb,IAAgB,EAAEc,KAAa,EAAEC,OAAiB,EAAEC,IAAiB;IAC5F,IAAIA,KAAKZ,MAAM,KAAK,GAAG;IACvB,MAAM,EAAEa,YAAY,EAAE,GAAG,MAAM5B;IAC/B,MAAM6B,WAAW,MAAMlB,KAAKmB,OAAO,CAAC,CAAC,kBAAkB,EAAEhC,WAAW2B,OAAO,CAAC,CAAC;IAC7E,MAAMM,kBAAkB,AAAE,CAAA,MAAMF,SAASG,GAAG,EAAC,EAA+Bf,GAAG,CAAC,CAACgB,IAAMA,EAAEX,IAAI;IAC7F,MAAMY,aAAa,IAAIC,IAAIT,QAAQT,GAAG,CAAC,CAACK,MAAMc,IAAM;YAACd;YAAMc;SAAE;IAE7D,MAAMC,SAAS,AAAC1B,KAA0B2B,MAAM;IAChD,MAAMC,WAAW,MAAMF,OAAOG,cAAc,CAACf;IAC7C,IAAI;QACF,KAAK,MAAMgB,OAAOd,KAAM;YACtB,KAAK,MAAML,QAAQS,gBAAiB;gBAClC,MAAMW,MAAMR,WAAWS,GAAG,CAACrB;gBAC3B,MAAMsB,QAAQF,QAAQG,YAAYA,YAAYJ,GAAG,CAACC,IAAI;gBACtD,IAAIA,QAAQG,WAAWN,SAASO,aAAa;qBACxC,IAAIF,UAAU,QAAQA,UAAUC,WAAWN,SAASQ,UAAU;qBAC9D,IAAIlD,yBAAyBmD,GAAG,CAAC1B,OAAOiB,SAASU,WAAW,CAACL;qBAC7DL,SAASW,aAAa,CAACtB,aAAagB;YAC3C;YACAL,SAASY,MAAM;QACjB;QACAZ,SAASa,SAAS;IACpB,SAAU;QACRb,SAASc,SAAS;IACpB;AACF;AAEA,OAAO,MAAMC,gBAAkC;IAC7CC,WAAW,IAAM;IACjBC,kBAAiBC,KAAK;QACpB,IAAIA,QAAQhD,yBAAyB;YACnC,MAAM,IAAIb,WAAW,gBAAgB,CAAC,uBAAuB,EAAE6D,MAAM,8CAA8C,EAAEhD,wBAAwB,gIAAgI,CAAC;QAChR;IACF;IACAU;IACAT;IACAc;AACF,EAAE"}

package/dist/esm/store/index.d.ts

@@ -1,15 +1,11 @@
 import type { ResolvedConfig } from '../config/index.js';
-import type { Builder } from './builder.js';
 import type { OpenResult } from './open.js';
 export declare function openStore(cfg: ResolvedConfig): Promise<OpenResult>;
-export declare function openStoreForWatch(cfg: ResolvedConfig): Promise<OpenResult & {
-    builder: Builder;
-}>;
-export declare function requireWatchConcurrency(cfg: ResolvedConfig): void;
 export { clearCache } from './cache.js';
 export type { OpenResult } from './open.js';
 export { docCount } from './open.js';
 export { getMeta, setMeta } from './shared.js';
 export { DB_FILENAME, SCHEMA_VERSION } from './sqlite/open.js';
+export type { Stages } from './stages.js';
 export type { Capability, Connection, DocumentStore, LexicalHit, LexicalIndex, LexicalQueryOptions, RawStatement, RunResult, SqlSession, Statement, Store, VectorCandidate, VectorSimilar, VectorStore, VectorWriteRow } from './types.js';
 export { CAPABILITY_NAMES } from './types.js';