sensemaking 0.17.0 → 0.18.0

package/dist/esm/config/validate.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\n// Exported so a docs test can assert every key here has a schema.json property -- one source\n// of truth for what the embed block accepts, checked against the other for drift.\nexport const KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages', 'chunkTokens']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in db.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n  if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that\n// doesn't need the top-level embed block: links is seeded by word-match rows, so declaring it\n// without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","provider","url","key","languages","some","l","chunkTokens","Number","isInteger","isSignalName","includes","isSignalWeight","isFinite","validatePresetSignals","map","signal","weight","names","requires","validatePreset","preset","exclude","signals","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AAGnF,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;CAAU;AACvG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,6FAA6F;AAC7F,kFAAkF;AAClF,OAAO,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;IAAa;CAAc,EAAE;AACzG,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,sBAAsBlB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACK,MAAMC,EAAE,IAAI1B,OAAO2B,OAAO,CAACT,OAAQ;QAC7C,IAAI,OAAOQ,MAAM,WAAW;YAC1B,MAAM,IAAItC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEW,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBtB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMe,QAAQvB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC4B,OAAO3B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIS,MAAMC,KAAK,KAAKR,aAAa,OAAOO,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIe,MAAMC,KAAK,KAAKR,aAAaO,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIe,MAAME,QAAQ,KAAKT,aAAaO,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIe,MAAMG,GAAG,KAAKV,aAAa,OAAOO,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIe,MAAMI,GAAG,KAAKX,aAAa,OAAOO,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIe,MAAME,QAAQ,KAAK,YAAaF,CAAAA,MAAMG,GAAG,KAAKV,aAAaO,MAAMG,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIe,MAAMK,SAAS,KAAKZ,aAAc,CAAA,CAACf,MAAMC,OAAO,CAACqB,MAAMK,SAAS,KAAKL,MAAMK,SAAS,CAACzB,MAAM,KAAK,KAAKoB,MAAMK,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;IACA,IAAIe,MAAMQ,WAAW,KAAKf,aAAc,CAAA,OAAOO,MAAMQ,WAAW,KAAK,YAAY,CAACC,OAAOC,SAAS,CAACV,MAAMQ,WAAW,KAAKR,MAAMQ,WAAW,IAAI,CAAA,GAAI;QAChJ,MAAM,IAAIjD,WAAW,kBAAkB,GAAG0B,WAAW,8CAA8C,CAAC;IACtG;AACF;AAEA,SAAS0B,aAAalC,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmCoD,QAAQ,CAACnC;AACnF;AAEA,SAASoC,eAAepC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYgC,OAAOK,QAAQ,CAACrC,UAAUA,QAAQ;AACxE;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,iCAAiC;AACjC,SAASsC,sBAAsBnB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEW,KAAK,2FAA2F,EAAEpC,aAAa+B,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMO,UAAU3B,OAAO2B,OAAO,CAACrB;IAC/B,MAAMa,UAAUQ,QAAQzB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACqC,aAAarC,IAAI0C,GAAG,CAAC,CAAC,CAAC1C,EAAE,GAAKA;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,iCAAiC,EAAEN,QAAQC,IAAI,CAAC,MAAM,oBAAoB,EAAE/B,aAAa+B,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAAC0B,QAAQC,OAAO,IAAIpB,QAAS;QACtC,IAAI,CAACe,eAAeK,SAAS;YAC3B,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,SAAS,EAAEqB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAIxD,IAAImC,QAAQkB,GAAG,CAAC,CAAC,CAAC1C,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAAC2C,QAAQG,SAAS,IAAIjD,OAAO2B,OAAO,CAACrC,sBAA0D;QACxG,IAAI0D,MAAM5C,GAAG,CAAC0C,WAAW,CAACE,MAAM5C,GAAG,CAAC6C,WAAW;YAC7C,MAAM,IAAI7D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,cAAc,EAAEqB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAExB,KAAK,mBAAmB,EAAEqB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAezB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAM0B,SAAS7C;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACkD,QAAQjD,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAACf,sBAAsB8C,OAAOnC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOC,OAAO,KAAK9B,aAAa,CAACjB,sBAAsB8C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOE,OAAO,KAAK/B,WAAWsB,sBAAsBnB,MAAM0B,OAAOE,OAAO,EAAEvC;IAC9E,IAAIqC,OAAOhD,CAAC,KAAKmB,aAAc,CAAA,OAAO6B,OAAOhD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACY,OAAOhD,CAAC,KAAKgD,OAAOhD,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAI0B,OAAOG,KAAK,KAAKhC,aAAa,OAAO6B,OAAOG,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIlE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAAS8B,mBAAmB9B,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uCAAuC,EAAE+B,KAAKC,SAAS,CAACnD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMoD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGpD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMkC,QAAQrD;IACd,IAAI,SAASqD,OAAO;QAClB,MAAMxC,UAAUnB,OAAOC,IAAI,CAAC0D,OAAOzD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOuC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACrC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYkC,OAAO;QACrB,MAAMxC,UAAUnB,OAAOC,IAAI,CAAC0D,OAAOzD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOuC,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACtC,IAAI,OAAO,IAAI;YAClE,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIkC,MAAMR,MAAM,KAAK7B,aAAc,CAAA,OAAOqC,MAAMR,MAAM,KAAK,YAAYQ,MAAMR,MAAM,CAAC1C,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIkC,MAAM3C,OAAO,KAAKM,aAAa,CAACjB,sBAAsBsD,MAAM3C,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAMP,OAAO,KAAK9B,aAAa,CAACjB,sBAAsBsD,MAAMP,OAAO,GAAG;YACxE,MAAM,IAAIhE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAML,KAAK,KAAKhC,aAAa,OAAOqC,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIlE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIkC,MAAMxD,CAAC,KAAKmB,aAAc,CAAA,OAAOqC,MAAMxD,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACoB,MAAMxD,CAAC,KAAKwD,MAAMxD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASqC,eAAejD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAIgE,MAAM,KAAKzC,WAAW;QAC5B,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMkD,UAAUjE,IAAIiE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYzD,MAAMC,OAAO,CAACwD,YAAYhE,OAAOC,IAAI,CAAC+D,SAASvD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACW,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACqC,SAAU;QACnDd,eAAezB,MAAMnB,OAAOQ;IAC9B;IACA,IAAIkD,QAAQC,OAAO,KAAK3C,WAAW;QACjC,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAImE,OAAO,KAAK5C,WAAWvB,IAAImE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOnE,IAAImE,OAAO,KAAK,YAAYnE,IAAImE,OAAO,KAAK,QAAQ3D,MAAMC,OAAO,CAACT,IAAImE,OAAO,GAAG;QACzF,MAAM,IAAI9E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMoD,UAAUnE,IAAImE,OAAO;IAC3B,KAAK,MAAM,CAACzC,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACuC,SAAU;QACnDX,mBAAmB9B,MAAMnB,OAAOQ;IAClC;IAEA,IAAIf,IAAIoE,QAAQ,KAAK7C,WAAW;QAC9BE,sBAAsBzB,IAAIoE,QAAQ,EAAErD;IACtC;IACA,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3BM,mBAAmB7B,IAAI8B,KAAK,EAAEf;IAChC;IACA,IAAIf,IAAIqE,OAAO,KAAK9C,WAAW;QAC7BL,qBAAqBlB,IAAIqE,OAAO,EAAEtD;IACpC;IAEA,8FAA8F;IAC9F,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3B,KAAK,MAAM,CAACG,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACqC,SAAU;YACnD,MAAMX,UAAU,AAAC/C,MAAkC+C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAO1B;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'store', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\n// Exported so a docs test can assert every key here has a schema.json property -- one source\n// of truth for what the embed block accepts, checked against the other for drift.\nexport const KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages', 'chunkTokens']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in store/sqlite/open.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateStoreKey(value: unknown, configPath: string): void {\n  if (value !== 'sqlite' && value !== 'duckdb') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: store must be \"sqlite\" or \"duckdb\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n  if (embed.chunkTokens !== undefined && (typeof embed.chunkTokens !== 'number' || !Number.isInteger(embed.chunkTokens) || embed.chunkTokens <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.chunkTokens must be a positive integer`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that\n// doesn't need the top-level embed block: links is seeded by word-match rows, so declaring it\n// without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n  if (cfg.store !== undefined) {\n    validateStoreKey(cfg.store, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateStoreKey","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","provider","url","key","languages","some","l","chunkTokens","Number","isInteger","isSignalName","includes","isSignalWeight","isFinite","validatePresetSignals","map","signal","weight","names","requires","validatePreset","preset","exclude","signals","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content","store"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AAGnF,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;IAAS;CAAU;AAChH,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,6FAA6F;AAC7F,kFAAkF;AAClF,OAAO,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;IAAa;CAAc,EAAE;AACzG,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,yGAAyG;AACzG,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,iBAAiBlB,KAAc,EAAEQ,UAAkB;IAC1D,IAAIR,UAAU,YAAYA,UAAU,UAAU;QAC5C,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oCAAoC,CAAC;IAC5F;AACF;AAEA,SAASW,sBAAsBnB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACM,MAAMC,EAAE,IAAI3B,OAAO4B,OAAO,CAACV,OAAQ;QAC7C,IAAI,OAAOS,MAAM,WAAW;YAC1B,MAAM,IAAIvC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEY,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBvB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMgB,QAAQxB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC6B,OAAO5B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIU,MAAMC,KAAK,KAAKT,aAAa,OAAOQ,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIgB,MAAMC,KAAK,KAAKT,aAAaQ,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIgB,MAAME,QAAQ,KAAKV,aAAaQ,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIgB,MAAMG,GAAG,KAAKX,aAAa,OAAOQ,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIgB,MAAMI,GAAG,KAAKZ,aAAa,OAAOQ,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI9C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIgB,MAAME,QAAQ,KAAK,YAAaF,CAAAA,MAAMG,GAAG,KAAKX,aAAaQ,MAAMG,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIgB,MAAMK,SAAS,KAAKb,aAAc,CAAA,CAACf,MAAMC,OAAO,CAACsB,MAAMK,SAAS,KAAKL,MAAMK,SAAS,CAAC1B,MAAM,KAAK,KAAKqB,MAAMK,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIjD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;IACA,IAAIgB,MAAMQ,WAAW,KAAKhB,aAAc,CAAA,OAAOQ,MAAMQ,WAAW,KAAK,YAAY,CAACC,OAAOC,SAAS,CAACV,MAAMQ,WAAW,KAAKR,MAAMQ,WAAW,IAAI,CAAA,GAAI;QAChJ,MAAM,IAAIlD,WAAW,kBAAkB,GAAG0B,WAAW,8CAA8C,CAAC;IACtG;AACF;AAEA,SAAS2B,aAAanC,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmCqD,QAAQ,CAACpC;AACnF;AAEA,SAASqC,eAAerC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYiC,OAAOK,QAAQ,CAACtC,UAAUA,QAAQ;AACxE;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,iCAAiC;AACjC,SAASuC,sBAAsBnB,IAAY,EAAEpB,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEY,KAAK,2FAA2F,EAAErC,aAAa+B,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMQ,UAAU5B,OAAO4B,OAAO,CAACtB;IAC/B,MAAMa,UAAUS,QAAQ1B,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACsC,aAAatC,IAAI2C,GAAG,CAAC,CAAC,CAAC3C,EAAE,GAAKA;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,iCAAiC,EAAEP,QAAQC,IAAI,CAAC,MAAM,oBAAoB,EAAE/B,aAAa+B,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAAC2B,QAAQC,OAAO,IAAIpB,QAAS;QACtC,IAAI,CAACe,eAAeK,SAAS;YAC3B,MAAM,IAAI5D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,SAAS,EAAEqB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAIzD,IAAIoC,QAAQkB,GAAG,CAAC,CAAC,CAAC3C,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAAC4C,QAAQG,SAAS,IAAIlD,OAAO4B,OAAO,CAACtC,sBAA0D;QACxG,IAAI2D,MAAM7C,GAAG,CAAC2C,WAAW,CAACE,MAAM7C,GAAG,CAAC8C,WAAW;YAC7C,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,cAAc,EAAEqB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAExB,KAAK,mBAAmB,EAAEqB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAezB,IAAY,EAAEpB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAM0B,SAAS9C;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACmD,QAAQlD,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,oBAAoB,EAAEP,QAAQC,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAACf,sBAAsB+C,OAAOpC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOC,OAAO,KAAK/B,aAAa,CAACjB,sBAAsB+C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAI0B,OAAOE,OAAO,KAAKhC,WAAWuB,sBAAsBnB,MAAM0B,OAAOE,OAAO,EAAExC;IAC9E,IAAIsC,OAAOjD,CAAC,KAAKmB,aAAc,CAAA,OAAO8B,OAAOjD,CAAC,KAAK,YAAY,CAACoC,OAAOC,SAAS,CAACY,OAAOjD,CAAC,KAAKiD,OAAOjD,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAI0B,OAAOG,KAAK,KAAKjC,aAAa,OAAO8B,OAAOG,KAAK,KAAK,UAAU;QAClE,MAAM,IAAInE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAAS8B,mBAAmB9B,IAAY,EAAEpB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,uCAAuC,EAAE+B,KAAKC,SAAS,CAACpD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMqD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGrD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMkC,QAAQtD;IACd,IAAI,SAASsD,OAAO;QAClB,MAAMzC,UAAUnB,OAAOC,IAAI,CAAC2D,OAAO1D,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,oBAAoB,EAAEP,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOwC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACtC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYkC,OAAO;QACrB,MAAMzC,UAAUnB,OAAOC,IAAI,CAAC2D,OAAO1D,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,oBAAoB,EAAEP,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOwC,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACvC,IAAI,OAAO,IAAI;YAClE,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIkC,MAAMR,MAAM,KAAK9B,aAAc,CAAA,OAAOsC,MAAMR,MAAM,KAAK,YAAYQ,MAAMR,MAAM,CAAC3C,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIkC,MAAM5C,OAAO,KAAKM,aAAa,CAACjB,sBAAsBuD,MAAM5C,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAMP,OAAO,KAAK/B,aAAa,CAACjB,sBAAsBuD,MAAMP,OAAO,GAAG;YACxE,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIkC,MAAML,KAAK,KAAKjC,aAAa,OAAOsC,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAInE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIkC,MAAMzD,CAAC,KAAKmB,aAAc,CAAA,OAAOsC,MAAMzD,CAAC,KAAK,YAAY,CAACoC,OAAOC,SAAS,CAACoB,MAAMzD,CAAC,KAAKyD,MAAMzD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAItC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASqC,eAAelD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAIiE,MAAM,KAAK1C,WAAW;QAC5B,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMmD,UAAUlE,IAAIkE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAY1D,MAAMC,OAAO,CAACyD,YAAYjE,OAAOC,IAAI,CAACgE,SAASxD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACY,MAAMpB,MAAM,IAAIN,OAAO4B,OAAO,CAACqC,SAAU;QACnDd,eAAezB,MAAMpB,OAAOQ;IAC9B;IACA,IAAImD,QAAQC,OAAO,KAAK5C,WAAW;QACjC,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAIoE,OAAO,KAAK7C,WAAWvB,IAAIoE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOpE,IAAIoE,OAAO,KAAK,YAAYpE,IAAIoE,OAAO,KAAK,QAAQ5D,MAAMC,OAAO,CAACT,IAAIoE,OAAO,GAAG;QACzF,MAAM,IAAI/E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMqD,UAAUpE,IAAIoE,OAAO;IAC3B,KAAK,MAAM,CAACzC,MAAMpB,MAAM,IAAIN,OAAO4B,OAAO,CAACuC,SAAU;QACnDX,mBAAmB9B,MAAMpB,OAAOQ;IAClC;IAEA,IAAIf,IAAIqE,QAAQ,KAAK9C,WAAW;QAC9BG,sBAAsB1B,IAAIqE,QAAQ,EAAEtD;IACtC;IACA,IAAIf,IAAI+B,KAAK,KAAKR,WAAW;QAC3BO,mBAAmB9B,IAAI+B,KAAK,EAAEhB;IAChC;IACA,IAAIf,IAAIsE,OAAO,KAAK/C,WAAW;QAC7BL,qBAAqBlB,IAAIsE,OAAO,EAAEvD;IACpC;IACA,IAAIf,IAAIuE,KAAK,KAAKhD,WAAW;QAC3BE,iBAAiBzB,IAAIuE,KAAK,EAAExD;IAC9B;IAEA,8FAA8F;IAC9F,IAAIf,IAAI+B,KAAK,KAAKR,WAAW;QAC3B,KAAK,MAAM,CAACI,MAAMpB,MAAM,IAAIN,OAAO4B,OAAO,CAACqC,SAAU;YACnD,MAAMX,UAAU,AAAChD,MAAkCgD,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAIlE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEY,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAO3B;AACT"}

package/dist/esm/embed/distribution.d.ts

@@ -0,0 +1,4 @@
+import type { Store } from '../store/types.js';
+export type LangCounts = Record<string, number>;
+export declare function languageDistribution(store: Store): Promise<LangCounts | undefined>;
+export declare function saveLanguageDistribution(store: Store, counts: LangCounts): Promise<void>;

package/dist/esm/embed/distribution.js

@@ -0,0 +1,11 @@
+import { getMeta, setMeta } from '../store/meta.js';
+// Split out of langfit.ts so reading the persisted counts (status.ts's only need) never pulls in
+// franc-min; only checkLanguageFit's classify path needs the classifier itself.
+const META_KEY = 'embed_languages';
+export async function languageDistribution(store) {
+    const raw = await getMeta(store, META_KEY);
+    return raw ? JSON.parse(raw) : undefined;
+}
+export async function saveLanguageDistribution(store, counts) {
+    await setMeta(store, META_KEY, JSON.stringify(counts));
+}

package/dist/esm/embed/distribution.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/distribution.ts"],"sourcesContent":["import { getMeta, setMeta } from '../store/meta.ts';\nimport type { Store } from '../store/types.ts';\n\n// Split out of langfit.ts so reading the persisted counts (status.ts's only need) never pulls in\n// franc-min; only checkLanguageFit's classify path needs the classifier itself.\nconst META_KEY = 'embed_languages';\n\nexport type LangCounts = Record<string, number>;\n\nexport async function languageDistribution(store: Store): Promise<LangCounts | undefined> {\n  const raw = await getMeta(store, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nexport async function saveLanguageDistribution(store: Store, counts: LangCounts): Promise<void> {\n  await setMeta(store, META_KEY, JSON.stringify(counts));\n}\n"],"names":["getMeta","setMeta","META_KEY","languageDistribution","store","raw","JSON","parse","undefined","saveLanguageDistribution","counts","stringify"],"mappings":"AAAA,SAASA,OAAO,EAAEC,OAAO,QAAQ,mBAAmB;AAGpD,iGAAiG;AACjG,gFAAgF;AAChF,MAAMC,WAAW;AAIjB,OAAO,eAAeC,qBAAqBC,KAAY;IACrD,MAAMC,MAAM,MAAML,QAAQI,OAAOF;IACjC,OAAOG,MAAMC,KAAKC,KAAK,CAACF,OAAOG;AACjC;AAEA,OAAO,eAAeC,yBAAyBL,KAAY,EAAEM,MAAkB;IAC7E,MAAMT,QAAQG,OAAOF,UAAUI,KAAKK,SAAS,CAACD;AAChD"}

package/dist/esm/embed/langfit.d.ts

@@ -1,6 +1,3 @@
-import type { DatabaseSync } from 'node:sqlite';
+import type { Store } from '../store/types.js';
 import type { EmbedProvider } from './types.js';
-type LangCounts = Record<string, number>;
-export declare function languageDistribution(db: DatabaseSync): LangCounts | undefined;
-export declare function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void>;
-export {};
+export declare function checkLanguageFit(store: Store, provider: EmbedProvider, texts: string[]): Promise<void>;

package/dist/esm/embed/langfit.js

@@ -1,34 +1,34 @@
-import { franc } from 'franc-min';
-import { getMeta, setMeta } from '../db/shared.js';
+import Module from 'node:module';
 import { SenseError } from '../errors.js';
+import { languageDistribution, saveLanguageDistribution } from './distribution.js';
 import { toIso3 } from './languages.js';
-const META_KEY = 'embed_languages';
 // franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a
 // per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).
 const SAMPLE_CHARS = 300;
 // A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule
 // never fires below this many classified chunks, declared languages or not.
 const MIN_CLASSIFIED = 10;
-export function languageDistribution(db) {
-    const raw = getMeta(db, META_KEY);
-    return raw ? JSON.parse(raw) : undefined;
-}
-function classify(text) {
+// franc-min is a 148K static import; a top-level import put it on every command's module graph
+// regardless of whether the semantic path ran, shipped as the 0.17.0 regression. Deferred to a
+// synchronous require inside checkLanguageFit so only the embed path pays for it.
+const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
+function classify(franc, text) {
     const code = franc(text.slice(0, SAMPLE_CHARS));
     return code === 'und' ? undefined : code;
 }
 // Merges chunk texts into the persisted distribution; throws (without persisting) if the model
 // declares languages and a clear majority classified so far is not among them.
-export async function checkLanguageFit(db, provider, texts) {
-    var _languageDistribution;
+export async function checkLanguageFit(store, provider, texts) {
+    var _ref;
     if (texts.length === 0) return;
-    const persisted = (_languageDistribution = languageDistribution(db)) !== null && _languageDistribution !== void 0 ? _languageDistribution : {};
+    const { franc } = _require('franc-min');
+    const persisted = (_ref = await languageDistribution(store)) !== null && _ref !== void 0 ? _ref : {};
     const merged = {
         ...persisted
     };
     for (const text of texts){
         var _merged_code;
-        const code = classify(text);
+        const code = classify(franc, text);
         if (code) merged[code] = ((_merged_code = merged[code]) !== null && _merged_code !== void 0 ? _merged_code : 0) + 1;
     }
     const total = Object.values(merged).reduce((a, b)=>a + b, 0);
@@ -40,5 +40,5 @@ export async function checkLanguageFit(db, provider, texts) {
             throw new SenseError('EMBED_MODEL_MISMATCH', `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is "${majorityLang}" (ISO 639-3), which is not among them; choose a model for this tree's languages: \`sense init --model ...\`, see the sense-setup skill or INTEGRATIONS.md`);
         }
     }
-    setMeta(db, META_KEY, JSON.stringify(merged));
+    await saveLanguageDistribution(store, merged);
 }

package/dist/esm/embed/langfit.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import type { DatabaseSync } from 'node:sqlite';\nimport { franc } from 'franc-min';\nimport { getMeta, setMeta } from '../db/shared.ts';\nimport { SenseError } from '../errors.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst META_KEY = 'embed_languages';\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\ntype LangCounts = Record<string, number>;\n\nexport function languageDistribution(db: DatabaseSync): LangCounts | undefined {\n  const raw = getMeta(db, META_KEY);\n  return raw ? JSON.parse(raw) : undefined;\n}\n\nfunction classify(text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(db: DatabaseSync, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const persisted = languageDistribution(db) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  setMeta(db, META_KEY, JSON.stringify(merged));\n}\n"],"names":["franc","getMeta","setMeta","SenseError","toIso3","META_KEY","SAMPLE_CHARS","MIN_CLASSIFIED","languageDistribution","db","raw","JSON","parse","undefined","classify","text","code","slice","checkLanguageFit","provider","texts","length","persisted","merged","total","Object","values","reduce","a","b","languages","declared","Set","map","majorityLang","majorityCount","entries","sort","has","pct","Math","round","id","join","stringify"],"mappings":"AACA,SAASA,KAAK,QAAQ,YAAY;AAClC,SAASC,OAAO,EAAEC,OAAO,QAAQ,kBAAkB;AACnD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,MAAM,QAAQ,iBAAiB;AAGxC,MAAMC,WAAW;AACjB,+FAA+F;AAC/F,8FAA8F;AAC9F,MAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,MAAMC,iBAAiB;AAIvB,OAAO,SAASC,qBAAqBC,EAAgB;IACnD,MAAMC,MAAMT,QAAQQ,IAAIJ;IACxB,OAAOK,MAAMC,KAAKC,KAAK,CAACF,OAAOG;AACjC;AAEA,SAASC,SAASC,IAAY;IAC5B,MAAMC,OAAOhB,MAAMe,KAAKE,KAAK,CAAC,GAAGX;IACjC,OAAOU,SAAS,QAAQH,YAAYG;AACtC;AAEA,+FAA+F;AAC/F,+EAA+E;AAC/E,OAAO,eAAeE,iBAAiBT,EAAgB,EAAEU,QAAuB,EAAEC,KAAe;QAE7EZ;IADlB,IAAIY,MAAMC,MAAM,KAAK,GAAG;IACxB,MAAMC,aAAYd,wBAAAA,qBAAqBC,iBAArBD,mCAAAA,wBAA4B,CAAC;IAC/C,MAAMe,SAAS;QAAE,GAAGD,SAAS;IAAC;IAC9B,KAAK,MAAMP,QAAQK,MAAO;YAEEG;QAD1B,MAAMP,OAAOF,SAASC;QACtB,IAAIC,MAAMO,MAAM,CAACP,KAAK,GAAG,EAACO,eAAAA,MAAM,CAACP,KAAK,cAAZO,0BAAAA,eAAgB,KAAK;IACjD;IAEA,MAAMC,QAAQC,OAAOC,MAAM,CAACH,QAAQI,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG;IAC5D,IAAIV,SAASW,SAAS,IAAIX,SAASW,SAAS,CAACT,MAAM,GAAG,KAAKG,SAASjB,gBAAgB;QAClF,MAAMwB,WAAW,IAAIC,IAAIb,SAASW,SAAS,CAACG,GAAG,CAAC7B;QAChD,MAAM,CAAC8B,cAAcC,cAAc,GAAGV,OAAOW,OAAO,CAACb,QAAQc,IAAI,CAAC,CAACT,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC3F,IAAIO,gBAAgBX,SAAS,OAAO,CAACO,SAASO,GAAG,CAACJ,eAAe;YAC/D,MAAMK,MAAMC,KAAKC,KAAK,CAAC,AAACN,gBAAgBX,QAAS;YACjD,MAAM,IAAIrB,WACR,wBACA,CAAC,YAAY,EAAEgB,SAASuB,EAAE,CAAC,qBAAqB,EAAEvB,SAASW,SAAS,CAACa,IAAI,CAAC,MAAM,OAAO,EAAEJ,IAAI,qCAAqC,EAAEL,aAAa,0JAA0J,CAAC;QAEhT;IACF;IACAhC,QAAQO,IAAIJ,UAAUM,KAAKiC,SAAS,CAACrB;AACvC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/langfit.ts"],"sourcesContent":["import Module from 'node:module';\nimport { SenseError } from '../errors.ts';\nimport type { Store } from '../store/types.ts';\nimport { languageDistribution, saveLanguageDistribution } from './distribution.ts';\nimport { toIso3 } from './languages.ts';\nimport type { EmbedProvider } from './types.ts';\n\n// franc's own reliability floor is a handful of characters; this is a cheap upper bound, not a\n// per-chunk minimum -- a short chunk is simply more likely to come back 'und' (unclassified).\nconst SAMPLE_CHARS = 300;\n// A majority computed over a handful of chunks is noise, not a corpus signal; the decision rule\n// never fires below this many classified chunks, declared languages or not.\nconst MIN_CLASSIFIED = 10;\n\n// franc-min is a 148K static import; a top-level import put it on every command's module graph\n// regardless of whether the semantic path ran, shipped as the 0.17.0 regression. Deferred to a\n// synchronous require inside checkLanguageFit so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\nfunction classify(franc: typeof import('franc-min')['franc'], text: string): string | undefined {\n  const code = franc(text.slice(0, SAMPLE_CHARS));\n  return code === 'und' ? undefined : code;\n}\n\n// Merges chunk texts into the persisted distribution; throws (without persisting) if the model\n// declares languages and a clear majority classified so far is not among them.\nexport async function checkLanguageFit(store: Store, provider: EmbedProvider, texts: string[]): Promise<void> {\n  if (texts.length === 0) return;\n  const { franc } = _require('franc-min') as typeof import('franc-min');\n  const persisted = (await languageDistribution(store)) ?? {};\n  const merged = { ...persisted };\n  for (const text of texts) {\n    const code = classify(franc, text);\n    if (code) merged[code] = (merged[code] ?? 0) + 1;\n  }\n\n  const total = Object.values(merged).reduce((a, b) => a + b, 0);\n  if (provider.languages && provider.languages.length > 0 && total >= MIN_CLASSIFIED) {\n    const declared = new Set(provider.languages.map(toIso3));\n    const [majorityLang, majorityCount] = Object.entries(merged).sort((a, b) => b[1] - a[1])[0];\n    if (majorityCount / total >= 0.5 && !declared.has(majorityLang)) {\n      const pct = Math.round((majorityCount / total) * 100);\n      throw new SenseError(\n        'EMBED_MODEL_MISMATCH',\n        `embed model ${provider.id} declares languages [${provider.languages.join(', ')}], but ${pct}% of this tree's classified text is \"${majorityLang}\" (ISO 639-3), which is not among them; choose a model for this tree's languages: \\`sense init --model ...\\`, see the sense-setup skill or INTEGRATIONS.md`\n      );\n    }\n  }\n  await saveLanguageDistribution(store, merged);\n}\n"],"names":["Module","SenseError","languageDistribution","saveLanguageDistribution","toIso3","SAMPLE_CHARS","MIN_CLASSIFIED","_require","require","createRequire","url","classify","franc","text","code","slice","undefined","checkLanguageFit","store","provider","texts","length","persisted","merged","total","Object","values","reduce","a","b","languages","declared","Set","map","majorityLang","majorityCount","entries","sort","has","pct","Math","round","id","join"],"mappings":"AAAA,OAAOA,YAAY,cAAc;AACjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,oBAAoB,EAAEC,wBAAwB,QAAQ,oBAAoB;AACnF,SAASC,MAAM,QAAQ,iBAAiB;AAGxC,+FAA+F;AAC/F,8FAA8F;AAC9F,MAAMC,eAAe;AACrB,gGAAgG;AAChG,4EAA4E;AAC5E,MAAMC,iBAAiB;AAEvB,+FAA+F;AAC/F,+FAA+F;AAC/F,kFAAkF;AAClF,MAAMC,WAAW,OAAOC,YAAY,cAAcR,OAAOS,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,SAASG,SAASC,KAA0C,EAAEC,IAAY;IACxE,MAAMC,OAAOF,MAAMC,KAAKE,KAAK,CAAC,GAAGV;IACjC,OAAOS,SAAS,QAAQE,YAAYF;AACtC;AAEA,+FAA+F;AAC/F,+EAA+E;AAC/E,OAAO,eAAeG,iBAAiBC,KAAY,EAAEC,QAAuB,EAAEC,KAAe;QAGxE;IAFnB,IAAIA,MAAMC,MAAM,KAAK,GAAG;IACxB,MAAM,EAAET,KAAK,EAAE,GAAGL,SAAS;IAC3B,MAAMe,aAAa,OAAA,MAAMpB,qBAAqBgB,oBAA3B,kBAAA,OAAsC,CAAC;IAC1D,MAAMK,SAAS;QAAE,GAAGD,SAAS;IAAC;IAC9B,KAAK,MAAMT,QAAQO,MAAO;YAEEG;QAD1B,MAAMT,OAAOH,SAASC,OAAOC;QAC7B,IAAIC,MAAMS,MAAM,CAACT,KAAK,GAAG,EAACS,eAAAA,MAAM,CAACT,KAAK,cAAZS,0BAAAA,eAAgB,KAAK;IACjD;IAEA,MAAMC,QAAQC,OAAOC,MAAM,CAACH,QAAQI,MAAM,CAAC,CAACC,GAAGC,IAAMD,IAAIC,GAAG;IAC5D,IAAIV,SAASW,SAAS,IAAIX,SAASW,SAAS,CAACT,MAAM,GAAG,KAAKG,SAASlB,gBAAgB;QAClF,MAAMyB,WAAW,IAAIC,IAAIb,SAASW,SAAS,CAACG,GAAG,CAAC7B;QAChD,MAAM,CAAC8B,cAAcC,cAAc,GAAGV,OAAOW,OAAO,CAACb,QAAQc,IAAI,CAAC,CAACT,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,CAAC,CAAC,EAAE;QAC3F,IAAIO,gBAAgBX,SAAS,OAAO,CAACO,SAASO,GAAG,CAACJ,eAAe;YAC/D,MAAMK,MAAMC,KAAKC,KAAK,CAAC,AAACN,gBAAgBX,QAAS;YACjD,MAAM,IAAIvB,WACR,wBACA,CAAC,YAAY,EAAEkB,SAASuB,EAAE,CAAC,qBAAqB,EAAEvB,SAASW,SAAS,CAACa,IAAI,CAAC,MAAM,OAAO,EAAEJ,IAAI,qCAAqC,EAAEL,aAAa,0JAA0J,CAAC;QAEhT;IACF;IACA,MAAM/B,yBAAyBe,OAAOK;AACxC"}

package/dist/esm/embed/query.d.ts

@@ -1,21 +1,14 @@
-import type { DatabaseSync } from 'node:sqlite';
 import type { Config } from '../config/index.js';
+import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void>;
-export declare function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{
-    path: string;
-    lines: string;
-    similarity: number;
-}>>;
-export declare function hasEmbedding(db: DatabaseSync, path: string): boolean;
-export declare function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: {
+export declare function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void>;
+export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
+export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;
     allowed?: Set<string>;
     k: number;
-}): Array<{
-    path: string;
-    similarity: number;
-}>;
+}): Promise<VectorSimilar[]>;

package/dist/esm/embed/query.js

@@ -5,11 +5,7 @@ import { progress } from '../output/progress.js';
 import { parseFile } from '../scan/index.js';
 import { checkLanguageFit } from './langfit.js';
 import { getProvider } from './registry.js';
-// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
-// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.
-const STORE_DIMS = 256;
-// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.
-const TARGET_CHUNK_CAP = 16;
+import { STORE_DIMS } from './types.js';
 // Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for
 // benchmark/lib/embed.mjs, which scores the same lever math offline.
 export function toStore(full, dims, int8) {
@@ -31,9 +27,9 @@ export function toStore(full, dims, int8) {
 }
 // Embed rows whose vector is NULL, re-deriving chunk text from the files through the
 // same parse + chunker that stored the rows.
-export async function embedPending(db, cfg, baseDir) {
+export async function embedPending(store, cfg, baseDir) {
     const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table
-    const dirty = db.prepare('SELECT "path", chunk FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk').all();
+    const dirty = await store.vectors.pending();
     if (dirty.length === 0) return;
     const storeDims = Math.min(STORE_DIMS, provider.dims);
     const byPath = new Map();
@@ -71,44 +67,36 @@ export async function embedPending(db, cfg, baseDir) {
     }
     // Over the exact texts about to be embedded, before any of them are: a mismatch fails
     // this run loudly instead of quietly storing vectors from the wrong model.
-    await checkLanguageFit(db, provider, jobs.map((j)=>j.text));
-    const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE "path" = ? AND chunk = ?');
+    await checkLanguageFit(store, provider, jobs.map((j)=>j.text));
     // The lazy build is the one long silence a first search hits (measured 23s at
     // 26k notes); progress makes it distinguishable from a hang.
     const report = progress('embedding chunks', jobs.length);
     for(let i = 0; i < jobs.length; i += provider.batchCap){
         const batch = jobs.slice(i, i + provider.batchCap);
         const vectors = await provider.embedDocuments(batch.map((j)=>j.text));
-        db.exec('BEGIN');
-        try {
-            batch.forEach((job, j)=>{
-                const { v, scale } = toStore(vectors[j], storeDims, true);
-                const q = new Int8Array(storeDims);
-                for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
-                update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);
-            });
-            db.exec('COMMIT');
-        } catch (err) {
-            db.exec('ROLLBACK');
-            throw err;
-        }
+        await store.vectors.writeVectors(batch.map((job, j)=>{
+            const { v, scale } = toStore(vectors[j], storeDims, true);
+            const q = new Int8Array(storeDims);
+            for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
+            return {
+                path: job.path,
+                chunk: job.chunk,
+                scale,
+                vector: Buffer.from(q.buffer)
+            };
+        }));
         report.tick(Math.min(i + provider.batchCap, jobs.length));
     }
     report.finish();
 }
-// Dequantised int8 dot products land a little either side of a true cosine, so an identical
-// pair prints 1.001 and undermines a column whose whole job is being a bounded number.
-function asCosine(score) {
-    return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;
-}
 // Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as
 // lexical syntax. Similarity comes back because the fused score cannot express match quality,
 // and nearest-neighbour search always returns a neighbour however far away.
-export async function semanticCandidates(db, cfg, terms, fetch, allowed) {
+export async function semanticCandidates(store, cfg, terms, fetch, allowed) {
     var _terms_match;
     const baseDir = cfg.baseDir;
     if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');
-    await embedPending(db, cfg, baseDir);
+    await embedPending(store, cfg, baseDir);
     const provider = await getProvider(cfg);
     const storeDims = Math.min(STORE_DIMS, provider.dims);
     const text = ((_terms_match = terms.match(/[\p{L}\p{N}]+/gu)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>![
@@ -118,64 +106,15 @@ export async function semanticCandidates(db, cfg, terms, fetch, allowed) {
             'NEAR'
         ].includes(t)).join(' ');
     const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);
-    const rows = db.prepare('SELECT "path", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
-    const best = new Map();
-    for (const row of rows){
-        if (allowed && !allowed.has(row.path)) continue;
-        const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));
-        let dot = 0;
-        for(let d = 0; d < q.length; d++)dot += q[d] * qv[d];
-        const score = dot * row.scale;
-        const existing = best.get(row.path);
-        if (!existing || score > existing.score) best.set(row.path, {
-            score,
-            lines: `L${row.start_line}-${row.end_line}`
-        });
-    }
-    return [
-        ...best.entries()
-    ].sort((a, b)=>b[1].score - a[1].score).slice(0, fetch).map(([path, b])=>({
-            path,
-            lines: b.lines,
-            similarity: asCosine(b.score)
-        }));
+    return store.vectors.candidates(qv, storeDims, fetch, allowed);
 }
 // Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
 // "this note has no text", which look the same in an empty result.
-export function hasEmbedding(db, path) {
-    const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE "path" = ? AND vector IS NOT NULL LIMIT 1').get(path);
-    return row !== undefined;
+export async function hasEmbedding(store, path) {
+    return store.vectors.hasVector(path);
 }
 // Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear
-// scan of stored vectors. Reads only what is stored, so it stays sync.
-export function similarNotes(db, _cfg, path, opts) {
-    // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus
-    // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.
-    const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE "path" = ? AND vector IS NOT NULL ORDER BY chunk').all(path);
-    if (targetRows.length === 0) return [];
-    const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));
-    const target = targetRows.filter((_, i)=>i % step === 0).map((row)=>({
-            v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength),
-            scale: row.scale
-        }));
-    const rows = db.prepare('SELECT "path", scale, vector FROM embeddings WHERE vector IS NOT NULL').all();
-    const best = new Map();
-    for (const row of rows){
-        if (row.path === path || opts.exclude.has(row.path) || opts.allowed && !opts.allowed.has(row.path)) continue;
-        const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);
-        for (const t of target){
-            let dot = 0;
-            const len = Math.min(t.v.length, other.length);
-            for(let d = 0; d < len; d++)dot += t.v[d] * other[d];
-            const score = dot * t.scale * row.scale;
-            const existing = best.get(row.path);
-            if (existing === undefined || score > existing) best.set(row.path, score);
-        }
-    }
-    return [
-        ...best.entries()
-    ].sort((a, b)=>b[1] - a[1]).slice(0, opts.k).map(([p, score])=>({
-            path: p,
-            similarity: asCosine(score)
-        }));
+// scan of stored vectors.
+export async function similarNotes(store, _cfg, path, opts) {
+    return store.vectors.similar(path, opts);
 }

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\n\n// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Queries stay f32 at the same dims.\nconst STORE_DIMS = 256;\n// Seed chunks that participate in a `related` scan; see similarNotes for the measurement.\nconst TARGET_CHUNK_CAP = 16;\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(db: DatabaseSync, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = db.prepare('SELECT \"path\", chunk FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk').all() as Array<{ path: string; chunk: number }>;\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    db,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  const update = db.prepare('UPDATE embeddings SET scale = ?, vector = ? WHERE \"path\" = ? AND chunk = ?');\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    db.exec('BEGIN');\n    try {\n      batch.forEach((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        update.run(scale, Buffer.from(q.buffer), job.path, job.chunk);\n      });\n      db.exec('COMMIT');\n    } catch (err) {\n      db.exec('ROLLBACK');\n      throw err;\n    }\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Dequantised int8 dot products land a little either side of a true cosine, so an identical\n// pair prints 1.001 and undermines a column whose whole job is being a bounded number.\nfunction asCosine(score: number): number {\n  return Math.round(Math.min(1, Math.max(-1, score)) * 1000) / 1000;\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(db: DatabaseSync, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<Array<{ path: string; lines: string; similarity: number }>> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(db, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  const rows = db.prepare('SELECT \"path\", start_line, end_line, scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{\n    path: string;\n    start_line: number;\n    end_line: number;\n    scale: number;\n    vector: Uint8Array;\n  }>;\n\n  const best = new Map<string, { score: number; lines: string }>();\n  for (const row of rows) {\n    if (allowed && !allowed.has(row.path)) continue;\n    const q = new Int8Array(row.vector.buffer, row.vector.byteOffset, Math.min(storeDims, row.vector.byteLength));\n    let dot = 0;\n    for (let d = 0; d < q.length; d++) dot += q[d] * qv[d];\n    const score = dot * row.scale;\n    const existing = best.get(row.path);\n    if (!existing || score > existing.score) best.set(row.path, { score, lines: `L${row.start_line}-${row.end_line}` });\n  }\n  return [...best.entries()]\n    .sort((a, b) => b[1].score - a[1].score)\n    .slice(0, fetch)\n    .map(([path, b]) => ({ path, lines: b.lines, similarity: asCosine(b.score) }));\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport function hasEmbedding(db: DatabaseSync, path: string): boolean {\n  const row = db.prepare('SELECT 1 AS ok FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL LIMIT 1').get(path) as { ok: number } | undefined;\n  return row !== undefined;\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors. Reads only what is stored, so it stays sync.\nexport function similarNotes(db: DatabaseSync, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Array<{ path: string; similarity: number }> {\n  // Cost is target_chunks x stored_chunks, so a heading-dense seed multiplies a full-corpus\n  // scan (12.7s at 201 chunks/note). Sample evenly, so late sections still get a vote.\n  const targetRows = db.prepare('SELECT scale, vector FROM embeddings WHERE \"path\" = ? AND vector IS NOT NULL ORDER BY chunk').all(path) as Array<{ scale: number; vector: Uint8Array }>;\n  if (targetRows.length === 0) return [];\n  const step = Math.max(1, Math.ceil(targetRows.length / TARGET_CHUNK_CAP));\n  const target = targetRows.filter((_, i) => i % step === 0).map((row) => ({ v: new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength), scale: row.scale }));\n\n  const rows = db.prepare('SELECT \"path\", scale, vector FROM embeddings WHERE vector IS NOT NULL').all() as Array<{ path: string; scale: number; vector: Uint8Array }>;\n  const best = new Map<string, number>();\n  for (const row of rows) {\n    if (row.path === path || opts.exclude.has(row.path) || (opts.allowed && !opts.allowed.has(row.path))) continue;\n    const other = new Int8Array(row.vector.buffer, row.vector.byteOffset, row.vector.byteLength);\n    for (const t of target) {\n      let dot = 0;\n      const len = Math.min(t.v.length, other.length);\n      for (let d = 0; d < len; d++) dot += t.v[d] * other[d];\n      const score = dot * t.scale * row.scale;\n      const existing = best.get(row.path);\n      if (existing === undefined || score > existing) best.set(row.path, score);\n    }\n  }\n\n  return [...best.entries()]\n    .sort((a, b) => b[1] - a[1])\n    .slice(0, opts.k)\n    .map(([p, score]) => ({ path: p, similarity: asCosine(score) }));\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","TARGET_CHUNK_CAP","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","db","cfg","baseDir","provider","dirty","prepare","all","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","update","report","i","batchCap","batch","slice","vectors","embedDocuments","exec","forEach","job","q","Int8Array","round","run","Buffer","from","buffer","err","tick","finish","asCosine","score","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","rows","best","has","vector","byteOffset","byteLength","dot","existing","lines","start_line","end_line","entries","sort","a","b","similarity","hasEmbedding","undefined","similarNotes","_cfg","opts","targetRows","step","ceil","target","_","exclude","other","len","k","p"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAGjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,QAAQ,mBAAmB;AAC7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAE5C,8FAA8F;AAC9F,6FAA6F;AAC7F,MAAMC,aAAa;AACnB,0FAA0F;AAC1F,MAAMC,mBAAmB;AAEzB,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAC/E,MAAMC,WAAW,MAAMpB,YAAYkB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQJ,GAAGK,OAAO,CAAC,oFAAoFC,GAAG;IAChH,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASxC,UAAU;gBAAEyC,SAASP;gBAAMQ,SAAS9C,KAAKyB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAEhD,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEsB,KAAK2B,GAAG,CAACC,SAAS,CAAClD,KAAK;QAClK,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMmD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMjD,iBACJkB,IACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,MAAMG,SAASlC,GAAGK,OAAO,CAAC;IAC1B,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAM8B,SAASvD,SAAS,oBAAoBuC,KAAKZ,MAAM;IACvD,IAAK,IAAI6B,IAAI,GAAGA,IAAIjB,KAAKZ,MAAM,EAAE6B,KAAKjC,SAASkC,QAAQ,CAAE;QACvD,MAAMC,QAAQnB,KAAKoB,KAAK,CAACH,GAAGA,IAAIjC,SAASkC,QAAQ;QACjD,MAAMG,UAAU,MAAMrC,SAASsC,cAAc,CAACH,MAAMN,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE/B,GAAG0C,IAAI,CAAC;QACR,IAAI;YACFJ,MAAMK,OAAO,CAAC,CAACC,KAAKX;gBAClB,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQsD,OAAO,CAACP,EAAE,EAAEzB,WAAW;gBACpD,MAAMqC,IAAI,IAAIC,UAAUtC;gBACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKoD,CAAC,CAACpD,EAAE,GAAGC,KAAKqD,KAAK,CAACzD,CAAC,CAACG,EAAE,GAAGG;gBAC7DsC,OAAOc,GAAG,CAACpD,OAAOqD,OAAOC,IAAI,CAACL,EAAEM,MAAM,GAAGP,IAAI7B,IAAI,EAAE6B,IAAI3B,KAAK;YAC9D;YACAjB,GAAG0C,IAAI,CAAC;QACV,EAAE,OAAOU,KAAK;YACZpD,GAAG0C,IAAI,CAAC;YACR,MAAMU;QACR;QACAjB,OAAOkB,IAAI,CAAC3D,KAAKe,GAAG,CAAC2B,IAAIjC,SAASkC,QAAQ,EAAElB,KAAKZ,MAAM;IACzD;IACA4B,OAAOmB,MAAM;AACf;AAEA,4FAA4F;AAC5F,uFAAuF;AACvF,SAASC,SAASC,KAAa;IAC7B,OAAO9D,KAAKqD,KAAK,CAACrD,KAAKe,GAAG,CAAC,GAAGf,KAAKG,GAAG,CAAC,CAAC,GAAG2D,UAAU,QAAQ;AAC/D;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBzD,EAAgB,EAAEC,GAAW,EAAEyD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAO3GF;IANd,MAAMxD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIxB,WAAW,eAAe;IAClD,MAAMqB,aAAaC,IAAIC,KAAKC;IAE5B,MAAMC,WAAW,MAAMpB,YAAYkB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACzB,YAAYmB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAAC2B,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAItF,IAAI,CAAC;IAClH,MAAM,EAAEa,GAAG2E,EAAE,EAAE,GAAG/E,QAAQ,MAAMiB,SAAS+D,UAAU,CAACnC,OAAOvB,WAAW;IAEtE,MAAM2D,OAAOnE,GAAGK,OAAO,CAAC,+FAA+FC,GAAG;IAQ1H,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIP,WAAW,CAACA,QAAQS,GAAG,CAACzD,IAAIG,IAAI,GAAG;QACvC,MAAM8B,IAAI,IAAIC,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE7E,KAAKe,GAAG,CAACD,WAAWI,IAAI0D,MAAM,CAACE,UAAU;QAC3G,IAAIC,MAAM;QACV,IAAK,IAAIhF,IAAI,GAAGA,IAAIoD,EAAEtC,MAAM,EAAEd,IAAKgF,OAAO5B,CAAC,CAACpD,EAAE,GAAGwE,EAAE,CAACxE,EAAE;QACtD,MAAM+D,QAAQiB,MAAM7D,IAAIhB,KAAK;QAC7B,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;QAClC,IAAI,CAAC2D,YAAYlB,QAAQkB,SAASlB,KAAK,EAAEY,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAE;YAAEyC;YAAOmB,OAAO,CAAC,CAAC,EAAE/D,IAAIgE,UAAU,CAAC,CAAC,EAAEhE,IAAIiE,QAAQ,EAAE;QAAC;IACnH;IACA,OAAO;WAAIT,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,CAACzB,KAAK,GAAGwB,CAAC,CAAC,EAAE,CAACxB,KAAK,EACtCjB,KAAK,CAAC,GAAGoB,OACT3B,GAAG,CAAC,CAAC,CAACjB,MAAMkE,EAAE,GAAM,CAAA;YAAElE;YAAM4D,OAAOM,EAAEN,KAAK;YAAEO,YAAY3B,SAAS0B,EAAEzB,KAAK;QAAE,CAAA;AAC/E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,SAAS2B,aAAanF,EAAgB,EAAEe,IAAY;IACzD,MAAMH,MAAMZ,GAAGK,OAAO,CAAC,kFAAkFS,GAAG,CAACC;IAC7G,OAAOH,QAAQwE;AACjB;AAEA,+FAA+F;AAC/F,uEAAuE;AACvE,OAAO,SAASC,aAAarF,EAAgB,EAAEsF,IAAY,EAAEvE,IAAY,EAAEwE,IAAgE;IACzI,0FAA0F;IAC1F,qFAAqF;IACrF,MAAMC,aAAaxF,GAAGK,OAAO,CAAC,+FAA+FC,GAAG,CAACS;IACjI,IAAIyE,WAAWjF,MAAM,KAAK,GAAG,OAAO,EAAE;IACtC,MAAMkF,OAAO/F,KAAKG,GAAG,CAAC,GAAGH,KAAKgG,IAAI,CAACF,WAAWjF,MAAM,GAAGtB;IACvD,MAAM0G,SAASH,WAAW1B,MAAM,CAAC,CAAC8B,GAAGxD,IAAMA,IAAIqD,SAAS,GAAGzD,GAAG,CAAC,CAACpB,MAAS,CAAA;YAAEtB,GAAG,IAAIwD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;YAAG5E,OAAOgB,IAAIhB,KAAK;QAAC,CAAA;IAE/K,MAAMuE,OAAOnE,GAAGK,OAAO,CAAC,yEAAyEC,GAAG;IACpG,MAAM8D,OAAO,IAAIzD;IACjB,KAAK,MAAMC,OAAOuD,KAAM;QACtB,IAAIvD,IAAIG,IAAI,KAAKA,QAAQwE,KAAKM,OAAO,CAACxB,GAAG,CAACzD,IAAIG,IAAI,KAAMwE,KAAK3B,OAAO,IAAI,CAAC2B,KAAK3B,OAAO,CAACS,GAAG,CAACzD,IAAIG,IAAI,GAAI;QACtG,MAAM+E,QAAQ,IAAIhD,UAAUlC,IAAI0D,MAAM,CAACnB,MAAM,EAAEvC,IAAI0D,MAAM,CAACC,UAAU,EAAE3D,IAAI0D,MAAM,CAACE,UAAU;QAC3F,KAAK,MAAMT,KAAK4B,OAAQ;YACtB,IAAIlB,MAAM;YACV,MAAMsB,MAAMrG,KAAKe,GAAG,CAACsD,EAAEzE,CAAC,CAACiB,MAAM,EAAEuF,MAAMvF,MAAM;YAC7C,IAAK,IAAId,IAAI,GAAGA,IAAIsG,KAAKtG,IAAKgF,OAAOV,EAAEzE,CAAC,CAACG,EAAE,GAAGqG,KAAK,CAACrG,EAAE;YACtD,MAAM+D,QAAQiB,MAAMV,EAAEnE,KAAK,GAAGgB,IAAIhB,KAAK;YACvC,MAAM8E,WAAWN,KAAKtD,GAAG,CAACF,IAAIG,IAAI;YAClC,IAAI2D,aAAaU,aAAa5B,QAAQkB,UAAUN,KAAKlD,GAAG,CAACN,IAAIG,IAAI,EAAEyC;QACrE;IACF;IAEA,OAAO;WAAIY,KAAKU,OAAO;KAAG,CACvBC,IAAI,CAAC,CAACC,GAAGC,IAAMA,CAAC,CAAC,EAAE,GAAGD,CAAC,CAAC,EAAE,EAC1BzC,KAAK,CAAC,GAAGgD,KAAKS,CAAC,EACfhE,GAAG,CAAC,CAAC,CAACiE,GAAGzC,MAAM,GAAM,CAAA;YAAEzC,MAAMkF;YAAGf,YAAY3B,SAASC;QAAO,CAAA;AACjE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","store","cfg","baseDir","provider","dirty","vectors","pending","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","report","i","batchCap","batch","slice","embedDocuments","writeVectors","job","q","Int8Array","round","vector","Buffer","from","buffer","tick","finish","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","candidates","hasEmbedding","hasVector","similarNotes","_cfg","opts","similar"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAEjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAASC,UAAU,QAAQ,aAAa;AAExC,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAAe;IAC3E,MAAMC,WAAW,MAAMnB,YAAYiB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQ,MAAMJ,MAAMK,OAAO,CAACC,OAAO;IACzC,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASvC,UAAU;gBAAEwC,SAASP;gBAAMQ,SAAS7C,KAAKwB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAE/C,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEqB,KAAK2B,GAAG,CAACC,SAAS,CAACjD,KAAK;QAClK,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMkD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMhD,iBACJiB,OACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAMG,SAASrD,SAAS,oBAAoBsC,KAAKZ,MAAM;IACvD,IAAK,IAAI4B,IAAI,GAAGA,IAAIhB,KAAKZ,MAAM,EAAE4B,KAAKhC,SAASiC,QAAQ,CAAE;QACvD,MAAMC,QAAQlB,KAAKmB,KAAK,CAACH,GAAGA,IAAIhC,SAASiC,QAAQ;QACjD,MAAM/B,UAAU,MAAMF,SAASoC,cAAc,CAACF,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE,MAAM/B,MAAMK,OAAO,CAACmC,YAAY,CAC9BH,MAAML,GAAG,CAAC,CAACS,KAAKR;YACd,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQmB,OAAO,CAAC4B,EAAE,EAAEzB,WAAW;YACpD,MAAMkC,IAAI,IAAIC,UAAUnC;YACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKiD,CAAC,CAACjD,EAAE,GAAGC,KAAKkD,KAAK,CAACtD,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEmB,MAAM0B,IAAI1B,IAAI;gBAAEE,OAAOwB,IAAIxB,KAAK;gBAAErB;gBAAOiD,QAAQC,OAAOC,IAAI,CAACL,EAAEM,MAAM;YAAE;QAClF;QAEFd,OAAOe,IAAI,CAACvD,KAAKe,GAAG,CAAC0B,IAAIhC,SAASiC,QAAQ,EAAEjB,KAAKZ,MAAM;IACzD;IACA2B,OAAOgB,MAAM;AACf;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBnD,KAAY,EAAEC,GAAW,EAAEmD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAOvGF;IANd,MAAMlD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIvB,WAAW,eAAe;IAClD,MAAMoB,aAAaC,OAAOC,KAAKC;IAE/B,MAAMC,WAAW,MAAMnB,YAAYiB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAACqB,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAI/E,IAAI,CAAC;IAClH,MAAM,EAAEY,GAAGqE,EAAE,EAAE,GAAGzE,QAAQ,MAAMiB,SAASyD,UAAU,CAAC7B,OAAOvB,WAAW;IAEtE,OAAOR,MAAMK,OAAO,CAACwD,UAAU,CAACF,IAAInD,WAAW6C,OAAOC;AACxD;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeQ,aAAa9D,KAAY,EAAEe,IAAY;IAC3D,OAAOf,MAAMK,OAAO,CAAC0D,SAAS,CAAChD;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiD,aAAahE,KAAY,EAAEiE,IAAY,EAAElD,IAAY,EAAEmD,IAAgE;IAC3I,OAAOlE,MAAMK,OAAO,CAAC8D,OAAO,CAACpD,MAAMmD;AACrC"}

package/dist/esm/embed/static.js

@@ -1,9 +1,13 @@
 import { existsSync, readFileSync } from 'node:fs';
+import Module from 'node:module';
 import { join } from 'node:path';
-import { Tokenizer } from '@huggingface/tokenizers';
 import { SenseError } from '../errors.js';
 import { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.js';
 const BATCH_CAP = 64;
+// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's
+// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.
+// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.
+const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
 // Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no
 // special tokens, drop unk ids, mean-pool, L2-normalize.
 export async function staticProvider(model) {
@@ -35,6 +39,7 @@ export async function staticProvider(model) {
     const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];
     const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));
     const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));
+    const { Tokenizer } = _require('@huggingface/tokenizers');
     const tok = new Tokenizer(tokenizerJson, {});
     const unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
     function one(text) {

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { Tokenizer } from '@huggingface/tokenizers';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","join","Tokenizer","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,SAAS,QAAQ,0BAA0B;AACpD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeC,eAAeC,KAAa;;QA8BLC,sBAA7BA,4BAAAA;IA7Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIV,eAAeO,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMV,cAAcQ,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASX,cAAcG;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMN,SAASI;QACf,KAAK,MAAMI,QAAQT,YAAa;YAC9B,IAAI,CAACR,WAAWE,KAAKa,KAAKE,QAAQ;gBAChC,MAAM,IAAIb,WAAW,uBAAuB,CAAC,YAAY,EAAES,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAER,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMiB,MAAMvB,aAAaC,KAAKa,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIlC,WAAW,eAAe,GAAGS,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAC7B,aAAaC,KAAKa,KAAK,mBAAmB;IAC3E,MAAMkC,MAAM,IAAI9C,UAAUW,eAAe,CAAC;IAC1C,MAAMoC,iBAAQpC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBqC,KAAK,cAA1BrC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBsC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOjC,OAAOkC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIf,aAAaN;QAC3B,IAAIe,IAAIjC,MAAM,KAAK,GAAG,OAAOuC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKnB;YACjB,IAAK,IAAIuB,IAAI,GAAGA,IAAIvB,MAAMuB,IAAKF,CAAC,CAACE,EAAE,IAAIlB,MAAM,CAACiB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIvB,MAAMuB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIjC,MAAM;YAClB0C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIvB,MAAMuB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE9C,OAAO;QAAE2B;QAAM2B,UAAUxD;QAAWK;QAAWoD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's\n// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.\n// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,gGAAgG;AAChG,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa;;QA+BLC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAId,eAAeW,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMd,cAAcY,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASf,cAAcO;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMV,SAASQ;QACf,KAAK,MAAMI,QAAQb,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKgB,KAAKE,QAAQ;gBAChC,MAAM,IAAIjB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAEZ,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMqB,MAAM3B,aAAaE,KAAKgB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAItC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAACjC,aAAaE,KAAKgB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAGzC,SAAS;IAC/B,MAAM0C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE/C,OAAO;QAAE2B;QAAM4B,UAAU7D;QAAWS;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}

package/dist/esm/embed/types.d.ts

@@ -1,3 +1,4 @@
+export declare const STORE_DIMS = 256;
 export interface EmbedProvider {
     id: string;
     dims: number;