sensemaking 0.15.4 → 0.16.0

package/dist/esm/config/validate.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'semantic', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\nconst KNOWN_EMBED_KEYS = new Set(['model', 'type', 'url', 'key']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in db.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, type?: \"static\"|\"api\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.type !== undefined && embed.type !== 'static' && embed.type !== 'api') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.type must be \"static\" or \"api\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, semantic, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.semantic !== undefined && typeof preset.semantic !== 'boolean') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.semantic must be a boolean`);\n  }\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, semantic?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","type","url","key","validatePreset","preset","exclude","semantic","Number","isInteger","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAG1C,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;CAAU;AACvG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAY;CAAQ;AAClF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAQ;IAAO;CAAM;AAChE,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI1B,WAAW,kBAAkB,GAAGwB,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAIjC,WAAW,kBAAkB,GAAGwB,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,sBAAsBlB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACK,MAAMC,EAAE,IAAI1B,OAAO2B,OAAO,CAACT,OAAQ;QAC7C,IAAI,OAAOQ,MAAM,WAAW;YAC1B,MAAM,IAAIpC,WAAW,kBAAkB,GAAGwB,WAAW,WAAW,EAAEW,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBtB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,0EAA0E,CAAC;IAClI;IACA,MAAMe,QAAQvB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC4B,OAAO3B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIS,MAAMC,KAAK,KAAKR,aAAa,OAAOO,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAIxC,WAAW,kBAAkB,GAAGwB,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIe,MAAMC,KAAK,KAAKR,aAAaO,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAIxC,WAAW,kBAAkB,GAAGwB,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIe,MAAME,IAAI,KAAKT,aAAaO,MAAME,IAAI,KAAK,YAAYF,MAAME,IAAI,KAAK,OAAO;QAC/E,MAAM,IAAIzC,WAAW,kBAAkB,GAAGwB,WAAW,sCAAsC,CAAC;IAC9F;IACA,IAAIe,MAAMG,GAAG,KAAKV,aAAa,OAAOO,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI1C,WAAW,kBAAkB,GAAGwB,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIe,MAAMI,GAAG,KAAKX,aAAa,OAAOO,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI3C,WAAW,kBAAkB,GAAGwB,WAAW,4BAA4B,CAAC;IACpF;AACF;AAEA,SAASoB,eAAeT,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAMU,SAAS7B;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACkC,QAAQjC,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,qDAAqD,CAAC;IACvK;IACA,IAAI,CAACf,sBAAsB8B,OAAOnB,OAAO,GAAG;QAC1C,MAAM,IAAI1B,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIU,OAAOC,OAAO,KAAKd,aAAa,CAACjB,sBAAsB8B,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAI9C,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIU,OAAOE,QAAQ,KAAKf,aAAa,OAAOa,OAAOE,QAAQ,KAAK,WAAW;QACzE,MAAM,IAAI/C,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,2BAA2B,CAAC;IACpG;IACA,IAAIU,OAAOhC,CAAC,KAAKmB,aAAc,CAAA,OAAOa,OAAOhC,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACJ,OAAOhC,CAAC,KAAKgC,OAAOhC,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIb,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAIU,OAAOK,KAAK,KAAKlB,aAAa,OAAOa,OAAOK,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIlD,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAASgB,mBAAmBhB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,uCAAuC,EAAEiB,KAAKC,SAAS,CAACrC,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMsC,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGtC,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIhB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMoB,QAAQvC;IACd,IAAI,SAASuC,OAAO;QAClB,MAAM1B,UAAUnB,OAAOC,IAAI,CAAC4C,OAAO3C,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOyB,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACvB,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAIjC,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYoB,OAAO;QACrB,MAAM1B,UAAUnB,OAAOC,IAAI,CAAC4C,OAAO3C,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOyB,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACxB,IAAI,OAAO,IAAI;YAClE,MAAM,IAAIjC,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIoB,MAAMV,MAAM,KAAKb,aAAc,CAAA,OAAOuB,MAAMV,MAAM,KAAK,YAAYU,MAAMV,MAAM,CAAC1B,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIoB,MAAM7B,OAAO,KAAKM,aAAa,CAACjB,sBAAsBwC,MAAM7B,OAAO,GAAG;YACxE,MAAM,IAAI1B,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIoB,MAAMT,OAAO,KAAKd,aAAa,CAACjB,sBAAsBwC,MAAMT,OAAO,GAAG;YACxE,MAAM,IAAI9C,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIoB,MAAML,KAAK,KAAKlB,aAAa,OAAOuB,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIlD,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIoB,MAAM1C,CAAC,KAAKmB,aAAc,CAAA,OAAOuB,MAAM1C,CAAC,KAAK,YAAY,CAACmC,OAAOC,SAAS,CAACM,MAAM1C,CAAC,KAAK0C,MAAM1C,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIb,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAInC,WAAW,kBAAkB,GAAGwB,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASuB,eAAenC,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIvB,WAAW,kBAAkB,GAAGwB,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAIkD,MAAM,KAAK3B,WAAW;QAC5B,MAAM,IAAIhC,WAAW,kBAAkB,GAAGwB,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMoC,UAAUnD,IAAImD,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAY3C,MAAMC,OAAO,CAAC0C,YAAYlD,OAAOC,IAAI,CAACiD,SAASzC,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAInB,WAAW,kBAAkB,GAAGwB,WAAW,mGAAmG,CAAC;IAC3J;IACA,KAAK,MAAM,CAACW,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACuB,SAAU;QACnDhB,eAAeT,MAAMnB,OAAOQ;IAC9B;IACA,IAAIoC,QAAQC,OAAO,KAAK7B,WAAW;QACjC,MAAM,IAAIhC,WAAW,kBAAkB,GAAGwB,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAIqD,OAAO,KAAK9B,WAAWvB,IAAIqD,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOrD,IAAIqD,OAAO,KAAK,YAAYrD,IAAIqD,OAAO,KAAK,QAAQ7C,MAAMC,OAAO,CAACT,IAAIqD,OAAO,GAAG;QACzF,MAAM,IAAI9D,WAAW,kBAAkB,GAAGwB,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMsC,UAAUrD,IAAIqD,OAAO;IAC3B,KAAK,MAAM,CAAC3B,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACyB,SAAU;QACnDX,mBAAmBhB,MAAMnB,OAAOQ;IAClC;IAEA,IAAIf,IAAIsD,QAAQ,KAAK/B,WAAW;QAC9BE,sBAAsBzB,IAAIsD,QAAQ,EAAEvC;IACtC;IACA,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3BM,mBAAmB7B,IAAI8B,KAAK,EAAEf;IAChC;IACA,IAAIf,IAAIuD,OAAO,KAAKhC,WAAW;QAC7BL,qBAAqBlB,IAAIuD,OAAO,EAAExC;IACpC;IAEA,OAAOf;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/config/validate.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { SIGNAL_NAMES, SIGNAL_PREREQUISITES, type SignalName } from './signals.ts';\nimport type { Config } from './types.ts';\n\n// Shape check for hand-edited files, so a typo names itself instead of surfacing as a\n// TypeError. Unknown top-level keys warn (forward compat); unknown keys inside a block error.\nconst KNOWN_KEYS = new Set(['$schema', 'version', 'presets', 'features', 'embed', 'content', 'queries']);\nconst KNOWN_PRESET_KEYS = new Set(['include', 'exclude', 'k', 'signals', 'where']);\nconst KNOWN_FEATURE_KEYS = new Set(['links', 'sections', 'tags', 'rank']);\nconst KNOWN_EMBED_KEYS = new Set(['model', 'provider', 'url', 'key', 'languages']);\nconst KNOWN_CONTENT_KEYS = new Set(['tokenize']);\nconst SAVED_SEARCH_KEYS = new Set(['search', 'preset', 'include', 'exclude', 'where', 'k']);\n\nexport function unknownConfigKeys(cfg: Record<string, unknown>): string[] {\n  return Object.keys(cfg).filter((k) => !KNOWN_KEYS.has(k));\n}\n\nfunction isNonEmptyStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.length > 0 && value.every((g) => typeof g === 'string' && g.length > 0);\n}\n\n// Pre-v3 shape check, just enough for migrateConfig to run safely (it reads scan.include\n// directly). The full v3 shape is checked by validateConfig once migration has produced it.\nexport function validateLegacyScan(parsed: unknown, configPath: string): void {\n  const cfg = typeof parsed === 'object' && parsed !== null && !Array.isArray(parsed) ? (parsed as Record<string, unknown>) : null;\n  const scan = cfg?.scan as { include?: unknown } | undefined;\n  if (!cfg || !scan || !isNonEmptyStringArray(scan.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: scan.include must be a non-empty array of glob strings`);\n  }\n}\n\n// Only the shape is checked here. Whether the linked SQLite accepts the tokenizer is settled\n// by probing it in db.ts, so this never has to carry a table of which version added what.\nfunction validateContentBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content must be an object of { tokenize?: string }`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_CONTENT_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content has unknown key(s) ${unknown.join(', ')}; content takes tokenize`);\n  }\n  if (block.tokenize !== undefined && (typeof block.tokenize !== 'string' || block.tokenize.trim() === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: content.tokenize must be a non-empty string, e.g. \"trigram\" or \"unicode61 tokenchars '-_'\"`);\n  }\n}\n\nfunction validateFeaturesBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features must be an object of name -> boolean`);\n  }\n  const block = value as Record<string, unknown>;\n  const unknown = Object.keys(block).filter((k) => !KNOWN_FEATURE_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: features has unknown key(s) ${unknown.join(', ')}; embed is not a features key -- see the top-level \"embed\" block`);\n  }\n  for (const [name, v] of Object.entries(block)) {\n    if (typeof v !== 'boolean') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: features.${name} must be a boolean`);\n    }\n  }\n}\n\nfunction validateEmbedBlock(value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed must be an object of { model?, provider?: \"static\"|\"openai\"|\"cohere\", url?, key? }`);\n  }\n  const embed = value as Record<string, unknown>;\n  const unknown = Object.keys(embed).filter((k) => !KNOWN_EMBED_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed has unknown key(s) ${unknown.join(', ')}`);\n  }\n  if (embed.model !== undefined && typeof embed.model !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model must be a string`);\n  }\n  // The block exists to name a model; provider settings without one would read as \"vectors\n  // configured\" while embedding stays off, which is the ambiguity v4 removed.\n  if (embed.model === undefined || embed.model === '') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.model is required when the \"embed\" block is present (it is what turns vectors on); remove the block to index without vectors`);\n  }\n  if (embed.provider !== undefined && embed.provider !== 'static' && embed.provider !== 'openai' && embed.provider !== 'cohere') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider must be \"static\", \"openai\", or \"cohere\"`);\n  }\n  if (embed.url !== undefined && typeof embed.url !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.url must be a string`);\n  }\n  if (embed.key !== undefined && typeof embed.key !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.key must be a string`);\n  }\n  // openai has no default host, so a missing url would otherwise only surface at the first\n  // vector search; a config error belongs at config time.\n  if (embed.provider === 'openai' && (embed.url === undefined || embed.url === '')) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.provider \"openai\" requires embed.url (e.g. \"http://localhost:11434/v1\" for Ollama)`);\n  }\n  if (embed.languages !== undefined && (!Array.isArray(embed.languages) || embed.languages.length === 0 || embed.languages.some((l) => typeof l !== 'string' || l === ''))) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: embed.languages must be a non-empty array of language tags (e.g. [\"en\", \"zh\"])`);\n  }\n}\n\nfunction isSignalName(value: unknown): value is SignalName {\n  return typeof value === 'string' && (SIGNAL_NAMES as readonly string[]).includes(value);\n}\n\nfunction isSignalWeight(value: unknown): value is number {\n  return typeof value === 'number' && Number.isFinite(value) && value > 0;\n}\n\n// Shape (an object of signal name -> weight) plus the one cross-signal prerequisite that\n// doesn't need the top-level embed block: links is seeded by word-match rows, so declaring it\n// without words is always empty.\nfunction validatePresetSignals(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value) || Object.keys(value).length === 0) {\n    throw new SenseError(\n      'CONFIG_INVALID',\n      `${configPath}: presets.${name}.signals must be a non-empty object of signal name -> weight (a finite number > 0), naming ${SIGNAL_NAMES.join(', ')} -- a preset that searches nothing is a mistake; name at least \"words\", or omit \"signals\" to use every signal whose prerequisites hold at weight 1`\n    );\n  }\n  const entries = Object.entries(value as Record<string, unknown>);\n  const unknown = entries.filter(([k]) => !isSignalName(k)).map(([k]) => k);\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals names unknown signal(s) ${unknown.join(', ')}; valid signals are ${SIGNAL_NAMES.join(', ')}`);\n  }\n  for (const [signal, weight] of entries) {\n    if (!isSignalWeight(weight)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals.${signal} must be a finite number > 0`);\n    }\n  }\n  const names = new Set(entries.map(([k]) => k));\n  for (const [signal, requires] of Object.entries(SIGNAL_PREREQUISITES) as Array<[SignalName, SignalName]>) {\n    if (names.has(signal) && !names.has(requires)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals has \"${signal}\" without \"${requires}\"; add \"${requires}\" to presets.${name}.signals, or drop \"${signal}\"`);\n    }\n  }\n}\n\nfunction validatePreset(name: string, value: unknown, configPath: string): void {\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} must be an object`);\n  }\n  const preset = value as Record<string, unknown>;\n  const unknown = Object.keys(preset).filter((k) => !KNOWN_PRESET_KEYS.has(k));\n  if (unknown.length > 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name} has unknown key(s) ${unknown.join(', ')}; a preset takes include, exclude, k, signals, where`);\n  }\n  if (!isNonEmptyStringArray(preset.include)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.include must be a non-empty array of glob strings`);\n  }\n  if (preset.exclude !== undefined && !isNonEmptyStringArray(preset.exclude)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.exclude must be a non-empty array of glob strings`);\n  }\n  if (preset.signals !== undefined) validatePresetSignals(name, preset.signals, configPath);\n  if (preset.k !== undefined && (typeof preset.k !== 'number' || !Number.isInteger(preset.k) || preset.k <= 0)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.k must be a positive integer`);\n  }\n  if (preset.where !== undefined && typeof preset.where !== 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.where must be a SQL condition string`);\n  }\n}\n\n// A queries.<name> entry: { sql } or a saved search { search, preset?, include?, exclude?, where?, k? }.\nfunction validateSavedQuery(name: string, value: unknown, configPath: string): void {\n  // A bare string used to mean SQL. It now fails rather than being inferred: an entry says\n  // which of the two verbs it runs, the same choice the CLI makes explicit.\n  if (typeof value === 'string') {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must say which verb it runs: { \"sql\": ${JSON.stringify(value.length > 40 ? `${value.slice(0, 37)}...` : value)} } to run it as SQL, or { \"search\": \"...\" } for a ranked search`);\n  }\n  if (typeof value !== 'object' || value === null || Array.isArray(value)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n  }\n  const entry = value as Record<string, unknown>;\n  if ('sql' in entry) {\n    const unknown = Object.keys(entry).filter((k) => k !== 'sql');\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a { sql } query takes only sql`);\n    }\n    if (typeof entry.sql !== 'string' || entry.sql.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.sql must be a non-empty string`);\n    }\n    return;\n  }\n  if ('search' in entry) {\n    const unknown = Object.keys(entry).filter((k) => !SAVED_SEARCH_KEYS.has(k));\n    if (unknown.length > 0) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} has unknown key(s) ${unknown.join(', ')}; a saved search takes search, preset, include, exclude, where, k`);\n    }\n    // A saved entry saves a question; a scope without a question is just flags.\n    if (typeof entry.search !== 'string' || entry.search.trim() === '') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.search must be non-empty text`);\n    }\n    if (entry.preset !== undefined && (typeof entry.preset !== 'string' || entry.preset.length === 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.preset must be a preset name`);\n    }\n    if (entry.include !== undefined && !isNonEmptyStringArray(entry.include)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.include must be a non-empty array of glob strings`);\n    }\n    if (entry.exclude !== undefined && !isNonEmptyStringArray(entry.exclude)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.exclude must be a non-empty array of glob strings`);\n    }\n    if (entry.where !== undefined && typeof entry.where !== 'string') {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.where must be a SQL condition string`);\n    }\n    if (entry.k !== undefined && (typeof entry.k !== 'number' || !Number.isInteger(entry.k) || entry.k <= 0)) {\n      throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name}.k must be a positive integer`);\n    }\n    return;\n  }\n  throw new SenseError('CONFIG_INVALID', `${configPath}: queries.${name} must be { sql } or { search, preset?, include?, exclude?, where?, k? }`);\n}\n\nexport function validateConfig(parsed: unknown, configPath: string): Config {\n  if (typeof parsed !== 'object' || parsed === null || Array.isArray(parsed)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: config must be a JSON object`);\n  }\n  const cfg = parsed as Record<string, unknown>;\n\n  // `checks` is rejected by name, not warned: silence would hide that assertions are gone.\n  if (cfg.checks !== undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: checks was removed in v3 -- sense check no longer asserts on saved queries; a returned row set is the reader's judgment`);\n  }\n\n  const presets = cfg.presets as Record<string, unknown> | undefined;\n  if (!presets || typeof presets !== 'object' || Array.isArray(presets) || Object.keys(presets).length === 0) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must be a non-empty object of preset name -> { include, exclude?, k?, signals?, where? }`);\n  }\n  for (const [name, value] of Object.entries(presets)) {\n    validatePreset(name, value, configPath);\n  }\n  if (presets.default === undefined) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: presets must include a \"default\" preset`);\n  }\n\n  if (cfg.queries === undefined) cfg.queries = {};\n  if (typeof cfg.queries !== 'object' || cfg.queries === null || Array.isArray(cfg.queries)) {\n    throw new SenseError('CONFIG_INVALID', `${configPath}: queries must be an object of name -> { sql } or { search }`);\n  }\n  const queries = cfg.queries as Record<string, unknown>;\n  for (const [name, value] of Object.entries(queries)) {\n    validateSavedQuery(name, value, configPath);\n  }\n\n  if (cfg.features !== undefined) {\n    validateFeaturesBlock(cfg.features, configPath);\n  }\n  if (cfg.embed !== undefined) {\n    validateEmbedBlock(cfg.embed, configPath);\n  }\n  if (cfg.content !== undefined) {\n    validateContentBlock(cfg.content, configPath);\n  }\n\n  // vectors' prerequisite is a named model, checked here once embed is known valid (or absent).\n  if (cfg.embed === undefined) {\n    for (const [name, value] of Object.entries(presets)) {\n      const signals = (value as Record<string, unknown>).signals as Record<string, unknown> | undefined;\n      if (signals && 'vectors' in signals) {\n        throw new SenseError('CONFIG_INVALID', `${configPath}: presets.${name}.signals includes \"vectors\", but no \"embed\" block names a model; add one, or drop \"vectors\" from presets.${name}.signals`);\n      }\n    }\n  }\n\n  return cfg as unknown as Config;\n}\n"],"names":["SenseError","SIGNAL_NAMES","SIGNAL_PREREQUISITES","KNOWN_KEYS","Set","KNOWN_PRESET_KEYS","KNOWN_FEATURE_KEYS","KNOWN_EMBED_KEYS","KNOWN_CONTENT_KEYS","SAVED_SEARCH_KEYS","unknownConfigKeys","cfg","Object","keys","filter","k","has","isNonEmptyStringArray","value","Array","isArray","length","every","g","validateLegacyScan","parsed","configPath","scan","include","validateContentBlock","block","unknown","join","tokenize","undefined","trim","validateFeaturesBlock","name","v","entries","validateEmbedBlock","embed","model","provider","url","key","languages","some","l","isSignalName","includes","isSignalWeight","Number","isFinite","validatePresetSignals","map","signal","weight","names","requires","validatePreset","preset","exclude","signals","isInteger","where","validateSavedQuery","JSON","stringify","slice","entry","sql","search","validateConfig","checks","presets","default","queries","features","content"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,YAAY,EAAEC,oBAAoB,QAAyB,eAAe;AAGnF,sFAAsF;AACtF,8FAA8F;AAC9F,MAAMC,aAAa,IAAIC,IAAI;IAAC;IAAW;IAAW;IAAW;IAAY;IAAS;IAAW;CAAU;AACvG,MAAMC,oBAAoB,IAAID,IAAI;IAAC;IAAW;IAAW;IAAK;IAAW;CAAQ;AACjF,MAAME,qBAAqB,IAAIF,IAAI;IAAC;IAAS;IAAY;IAAQ;CAAO;AACxE,MAAMG,mBAAmB,IAAIH,IAAI;IAAC;IAAS;IAAY;IAAO;IAAO;CAAY;AACjF,MAAMI,qBAAqB,IAAIJ,IAAI;IAAC;CAAW;AAC/C,MAAMK,oBAAoB,IAAIL,IAAI;IAAC;IAAU;IAAU;IAAW;IAAW;IAAS;CAAI;AAE1F,OAAO,SAASM,kBAAkBC,GAA4B;IAC5D,OAAOC,OAAOC,IAAI,CAACF,KAAKG,MAAM,CAAC,CAACC,IAAM,CAACZ,WAAWa,GAAG,CAACD;AACxD;AAEA,SAASE,sBAAsBC,KAAc;IAC3C,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,MAAM,GAAG,KAAKH,MAAMI,KAAK,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,EAAEF,MAAM,GAAG;AAC5G;AAEA,yFAAyF;AACzF,4FAA4F;AAC5F,OAAO,SAASG,mBAAmBC,MAAe,EAAEC,UAAkB;IACpE,MAAMf,MAAM,OAAOc,WAAW,YAAYA,WAAW,QAAQ,CAACN,MAAMC,OAAO,CAACK,UAAWA,SAAqC;IAC5H,MAAME,OAAOhB,gBAAAA,0BAAAA,IAAKgB,IAAI;IACtB,IAAI,CAAChB,OAAO,CAACgB,QAAQ,CAACV,sBAAsBU,KAAKC,OAAO,GAAG;QACzD,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;AACF;AAEA,6FAA6F;AAC7F,0FAA0F;AAC1F,SAASG,qBAAqBX,KAAc,EAAEQ,UAAkB;IAC9D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,oDAAoD,CAAC;IAC5G;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACP,mBAAmBQ,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,6BAA6B,EAAEK,QAAQC,IAAI,CAAC,MAAM,wBAAwB,CAAC;IAClI;IACA,IAAIF,MAAMG,QAAQ,KAAKC,aAAc,CAAA,OAAOJ,MAAMG,QAAQ,KAAK,YAAYH,MAAMG,QAAQ,CAACE,IAAI,OAAO,EAAC,GAAI;QACxG,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,4FAA4F,CAAC;IACpJ;AACF;AAEA,SAASU,sBAAsBlB,KAAc,EAAEQ,UAAkB;IAC/D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,+CAA+C,CAAC;IACvG;IACA,MAAMI,QAAQZ;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAACiB,OAAOhB,MAAM,CAAC,CAACC,IAAM,CAACT,mBAAmBU,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,EAAEK,QAAQC,IAAI,CAAC,MAAM,gEAAgE,CAAC;IAC3K;IACA,KAAK,MAAM,CAACK,MAAMC,EAAE,IAAI1B,OAAO2B,OAAO,CAACT,OAAQ;QAC7C,IAAI,OAAOQ,MAAM,WAAW;YAC1B,MAAM,IAAItC,WAAW,kBAAkB,GAAG0B,WAAW,WAAW,EAAEW,KAAK,kBAAkB,CAAC;QAC5F;IACF;AACF;AAEA,SAASG,mBAAmBtB,KAAc,EAAEQ,UAAkB;IAC5D,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,MAAMe,QAAQvB;IACd,MAAMa,UAAUnB,OAAOC,IAAI,CAAC4B,OAAO3B,MAAM,CAAC,CAACC,IAAM,CAACR,iBAAiBS,GAAG,CAACD;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,2BAA2B,EAAEK,QAAQC,IAAI,CAAC,OAAO;IACxG;IACA,IAAIS,MAAMC,KAAK,KAAKR,aAAa,OAAOO,MAAMC,KAAK,KAAK,UAAU;QAChE,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,yFAAyF;IACzF,4EAA4E;IAC5E,IAAIe,MAAMC,KAAK,KAAKR,aAAaO,MAAMC,KAAK,KAAK,IAAI;QACnD,MAAM,IAAI1C,WAAW,kBAAkB,GAAG0B,WAAW,oIAAoI,CAAC;IAC5L;IACA,IAAIe,MAAME,QAAQ,KAAKT,aAAaO,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,YAAYF,MAAME,QAAQ,KAAK,UAAU;QAC7H,MAAM,IAAI3C,WAAW,kBAAkB,GAAG0B,WAAW,wDAAwD,CAAC;IAChH;IACA,IAAIe,MAAMG,GAAG,KAAKV,aAAa,OAAOO,MAAMG,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,IAAIe,MAAMI,GAAG,KAAKX,aAAa,OAAOO,MAAMI,GAAG,KAAK,UAAU;QAC5D,MAAM,IAAI7C,WAAW,kBAAkB,GAAG0B,WAAW,4BAA4B,CAAC;IACpF;IACA,yFAAyF;IACzF,wDAAwD;IACxD,IAAIe,MAAME,QAAQ,KAAK,YAAaF,CAAAA,MAAMG,GAAG,KAAKV,aAAaO,MAAMG,GAAG,KAAK,EAAC,GAAI;QAChF,MAAM,IAAI5C,WAAW,kBAAkB,GAAG0B,WAAW,0FAA0F,CAAC;IAClJ;IACA,IAAIe,MAAMK,SAAS,KAAKZ,aAAc,CAAA,CAACf,MAAMC,OAAO,CAACqB,MAAMK,SAAS,KAAKL,MAAMK,SAAS,CAACzB,MAAM,KAAK,KAAKoB,MAAMK,SAAS,CAACC,IAAI,CAAC,CAACC,IAAM,OAAOA,MAAM,YAAYA,MAAM,GAAE,GAAI;QACxK,MAAM,IAAIhD,WAAW,kBAAkB,GAAG0B,WAAW,gFAAgF,CAAC;IACxI;AACF;AAEA,SAASuB,aAAa/B,KAAc;IAClC,OAAO,OAAOA,UAAU,YAAY,AAACjB,aAAmCiD,QAAQ,CAAChC;AACnF;AAEA,SAASiC,eAAejC,KAAc;IACpC,OAAO,OAAOA,UAAU,YAAYkC,OAAOC,QAAQ,CAACnC,UAAUA,QAAQ;AACxE;AAEA,yFAAyF;AACzF,8FAA8F;AAC9F,iCAAiC;AACjC,SAASoC,sBAAsBjB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC7E,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,UAAUN,OAAOC,IAAI,CAACK,OAAOG,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WACR,kBACA,GAAG0B,WAAW,UAAU,EAAEW,KAAK,2FAA2F,EAAEpC,aAAa+B,IAAI,CAAC,MAAM,kJAAkJ,CAAC;IAE3S;IACA,MAAMO,UAAU3B,OAAO2B,OAAO,CAACrB;IAC/B,MAAMa,UAAUQ,QAAQzB,MAAM,CAAC,CAAC,CAACC,EAAE,GAAK,CAACkC,aAAalC,IAAIwC,GAAG,CAAC,CAAC,CAACxC,EAAE,GAAKA;IACvE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,iCAAiC,EAAEN,QAAQC,IAAI,CAAC,MAAM,oBAAoB,EAAE/B,aAAa+B,IAAI,CAAC,OAAO;IAC7K;IACA,KAAK,MAAM,CAACwB,QAAQC,OAAO,IAAIlB,QAAS;QACtC,IAAI,CAACY,eAAeM,SAAS;YAC3B,MAAM,IAAIzD,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,SAAS,EAAEmB,OAAO,4BAA4B,CAAC;QACvH;IACF;IACA,MAAME,QAAQ,IAAItD,IAAImC,QAAQgB,GAAG,CAAC,CAAC,CAACxC,EAAE,GAAKA;IAC3C,KAAK,MAAM,CAACyC,QAAQG,SAAS,IAAI/C,OAAO2B,OAAO,CAACrC,sBAA0D;QACxG,IAAIwD,MAAM1C,GAAG,CAACwC,WAAW,CAACE,MAAM1C,GAAG,CAAC2C,WAAW;YAC7C,MAAM,IAAI3D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,cAAc,EAAEmB,OAAO,WAAW,EAAEG,SAAS,QAAQ,EAAEA,SAAS,aAAa,EAAEtB,KAAK,mBAAmB,EAAEmB,OAAO,CAAC,CAAC;QAC1L;IACF;AACF;AAEA,SAASI,eAAevB,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IACtE,IAAI,OAAOR,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kBAAkB,CAAC;IAC3F;IACA,MAAMwB,SAAS3C;IACf,MAAMa,UAAUnB,OAAOC,IAAI,CAACgD,QAAQ/C,MAAM,CAAC,CAACC,IAAM,CAACV,kBAAkBW,GAAG,CAACD;IACzE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,oDAAoD,CAAC;IACtK;IACA,IAAI,CAACf,sBAAsB4C,OAAOjC,OAAO,GAAG;QAC1C,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIwB,OAAOC,OAAO,KAAK5B,aAAa,CAACjB,sBAAsB4C,OAAOC,OAAO,GAAG;QAC1E,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;IAC3H;IACA,IAAIwB,OAAOE,OAAO,KAAK7B,WAAWoB,sBAAsBjB,MAAMwB,OAAOE,OAAO,EAAErC;IAC9E,IAAImC,OAAO9C,CAAC,KAAKmB,aAAc,CAAA,OAAO2B,OAAO9C,CAAC,KAAK,YAAY,CAACqC,OAAOY,SAAS,CAACH,OAAO9C,CAAC,KAAK8C,OAAO9C,CAAC,IAAI,CAAA,GAAI;QAC5G,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;IACtG;IACA,IAAIwB,OAAOI,KAAK,KAAK/B,aAAa,OAAO2B,OAAOI,KAAK,KAAK,UAAU;QAClE,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;IAC9G;AACF;AAEA,yGAAyG;AACzG,SAAS6B,mBAAmB7B,IAAY,EAAEnB,KAAc,EAAEQ,UAAkB;IAC1E,yFAAyF;IACzF,0EAA0E;IAC1E,IAAI,OAAOR,UAAU,UAAU;QAC7B,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uCAAuC,EAAE8B,KAAKC,SAAS,CAAClD,MAAMG,MAAM,GAAG,KAAK,GAAGH,MAAMmD,KAAK,CAAC,GAAG,IAAI,GAAG,CAAC,GAAGnD,OAAO,+DAA+D,CAAC;IACxP;IACA,IAAI,OAAOA,UAAU,YAAYA,UAAU,QAAQC,MAAMC,OAAO,CAACF,QAAQ;QACvE,MAAM,IAAIlB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;IAChJ;IACA,MAAMiC,QAAQpD;IACd,IAAI,SAASoD,OAAO;QAClB,MAAMvC,UAAUnB,OAAOC,IAAI,CAACyD,OAAOxD,MAAM,CAAC,CAACC,IAAMA,MAAM;QACvD,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,gCAAgC,CAAC;QAClJ;QACA,IAAI,OAAOsC,MAAMC,GAAG,KAAK,YAAYD,MAAMC,GAAG,CAACpC,IAAI,OAAO,IAAI;YAC5D,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,+BAA+B,CAAC;QACxG;QACA;IACF;IACA,IAAI,YAAYiC,OAAO;QACrB,MAAMvC,UAAUnB,OAAOC,IAAI,CAACyD,OAAOxD,MAAM,CAAC,CAACC,IAAM,CAACN,kBAAkBO,GAAG,CAACD;QACxE,IAAIgB,QAAQV,MAAM,GAAG,GAAG;YACtB,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,oBAAoB,EAAEN,QAAQC,IAAI,CAAC,MAAM,iEAAiE,CAAC;QACnL;QACA,4EAA4E;QAC5E,IAAI,OAAOsC,MAAME,MAAM,KAAK,YAAYF,MAAME,MAAM,CAACrC,IAAI,OAAO,IAAI;YAClE,MAAM,IAAInC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,8BAA8B,CAAC;QACvG;QACA,IAAIiC,MAAMT,MAAM,KAAK3B,aAAc,CAAA,OAAOoC,MAAMT,MAAM,KAAK,YAAYS,MAAMT,MAAM,CAACxC,MAAM,KAAK,CAAA,GAAI;YACjG,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA,IAAIiC,MAAM1C,OAAO,KAAKM,aAAa,CAACjB,sBAAsBqD,MAAM1C,OAAO,GAAG;YACxE,MAAM,IAAI5B,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIiC,MAAMR,OAAO,KAAK5B,aAAa,CAACjB,sBAAsBqD,MAAMR,OAAO,GAAG;YACxE,MAAM,IAAI9D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,kDAAkD,CAAC;QAC3H;QACA,IAAIiC,MAAML,KAAK,KAAK/B,aAAa,OAAOoC,MAAML,KAAK,KAAK,UAAU;YAChE,MAAM,IAAIjE,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,qCAAqC,CAAC;QAC9G;QACA,IAAIiC,MAAMvD,CAAC,KAAKmB,aAAc,CAAA,OAAOoC,MAAMvD,CAAC,KAAK,YAAY,CAACqC,OAAOY,SAAS,CAACM,MAAMvD,CAAC,KAAKuD,MAAMvD,CAAC,IAAI,CAAA,GAAI;YACxG,MAAM,IAAIf,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,6BAA6B,CAAC;QACtG;QACA;IACF;IACA,MAAM,IAAIrC,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,uEAAuE,CAAC;AAChJ;AAEA,OAAO,SAASoC,eAAehD,MAAe,EAAEC,UAAkB;IAChE,IAAI,OAAOD,WAAW,YAAYA,WAAW,QAAQN,MAAMC,OAAO,CAACK,SAAS;QAC1E,MAAM,IAAIzB,WAAW,kBAAkB,GAAG0B,WAAW,8BAA8B,CAAC;IACtF;IACA,MAAMf,MAAMc;IAEZ,yFAAyF;IACzF,IAAId,IAAI+D,MAAM,KAAKxC,WAAW;QAC5B,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yHAAyH,CAAC;IACjL;IAEA,MAAMiD,UAAUhE,IAAIgE,OAAO;IAC3B,IAAI,CAACA,WAAW,OAAOA,YAAY,YAAYxD,MAAMC,OAAO,CAACuD,YAAY/D,OAAOC,IAAI,CAAC8D,SAAStD,MAAM,KAAK,GAAG;QAC1G,MAAM,IAAIrB,WAAW,kBAAkB,GAAG0B,WAAW,kGAAkG,CAAC;IAC1J;IACA,KAAK,MAAM,CAACW,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACoC,SAAU;QACnDf,eAAevB,MAAMnB,OAAOQ;IAC9B;IACA,IAAIiD,QAAQC,OAAO,KAAK1C,WAAW;QACjC,MAAM,IAAIlC,WAAW,kBAAkB,GAAG0B,WAAW,yCAAyC,CAAC;IACjG;IAEA,IAAIf,IAAIkE,OAAO,KAAK3C,WAAWvB,IAAIkE,OAAO,GAAG,CAAC;IAC9C,IAAI,OAAOlE,IAAIkE,OAAO,KAAK,YAAYlE,IAAIkE,OAAO,KAAK,QAAQ1D,MAAMC,OAAO,CAACT,IAAIkE,OAAO,GAAG;QACzF,MAAM,IAAI7E,WAAW,kBAAkB,GAAG0B,WAAW,4DAA4D,CAAC;IACpH;IACA,MAAMmD,UAAUlE,IAAIkE,OAAO;IAC3B,KAAK,MAAM,CAACxC,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACsC,SAAU;QACnDX,mBAAmB7B,MAAMnB,OAAOQ;IAClC;IAEA,IAAIf,IAAImE,QAAQ,KAAK5C,WAAW;QAC9BE,sBAAsBzB,IAAImE,QAAQ,EAAEpD;IACtC;IACA,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3BM,mBAAmB7B,IAAI8B,KAAK,EAAEf;IAChC;IACA,IAAIf,IAAIoE,OAAO,KAAK7C,WAAW;QAC7BL,qBAAqBlB,IAAIoE,OAAO,EAAErD;IACpC;IAEA,8FAA8F;IAC9F,IAAIf,IAAI8B,KAAK,KAAKP,WAAW;QAC3B,KAAK,MAAM,CAACG,MAAMnB,MAAM,IAAIN,OAAO2B,OAAO,CAACoC,SAAU;YACnD,MAAMZ,UAAU,AAAC7C,MAAkC6C,OAAO;YAC1D,IAAIA,WAAW,aAAaA,SAAS;gBACnC,MAAM,IAAI/D,WAAW,kBAAkB,GAAG0B,WAAW,UAAU,EAAEW,KAAK,yGAAyG,EAAEA,KAAK,QAAQ,CAAC;YACjM;QACF;IACF;IAEA,OAAO1B;AACT"}

package/dist/esm/db/open.d.ts

@@ -1,7 +1,7 @@
 import { DatabaseSync } from 'node:sqlite';
 import type { ResolvedConfig } from '../config/index.js';
 export declare const DB_FILENAME = "cache.db";
-export declare const SCHEMA_VERSION = "16";
+export declare const SCHEMA_VERSION = "17";
 export interface OpenResult {
     db: DatabaseSync;
     cfg: ResolvedConfig;

package/dist/esm/db/open.js

@@ -7,12 +7,12 @@ import { contentTokenize, featureSignature, STATE_DIR } from '../config/index.js
 import { SenseError } from '../errors.js';
 import { activeFeatures } from '../features/index.js';
 import { registerFunctions } from '../sql-functions.js';
-import { changedSignatureKeys, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
+import { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.js';
 import { getMeta, setMeta } from './shared.js';
 export const DB_FILENAME = 'cache.db';
 // Cache shape version, independent of the config's own `version`. Bumping it rebuilds
 // existing trees on first query.
-export const SCHEMA_VERSION = '16';
+export const SCHEMA_VERSION = '17';
 // Stemming is English-only, but the segmentation underneath it is what decides coverage:
 // unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)
 // indexes a whole run as one token and word search finds nothing. `content.tokenize` is how
@@ -107,6 +107,11 @@ export function open(cfg) {
             console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');
             db.exec('DROP TABLE content');
             tokenizeOnlyRebuild = true;
+        } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features !== null && features !== void 0 ? features : '', wantFeatures)) {
+            // First sight of a resolved weight identity: the model itself hasn't changed, so
+            // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.
+            console.error("sense: recorded the embedding model's resolved identity; vectors are unaffected");
+            setMeta(db, 'features', wantFeatures);
         } else {
             const changed = signatureDiff(features !== null && features !== void 0 ? features : '', wantFeatures);
             console.error(`sense: config change (${changed}) rebuilds the index`);

package/dist/esm/db/open.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport { registerFunctions } from '../sql-functions.ts';\nimport { changedSignatureKeys, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { getMeta, setMeta } from './shared.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '16';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config, tokenize: string): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db, contentTokenize(cfg) === undefined);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n  // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n  const tokenize = resolveTokenize(db, cfg);\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  let tokenizeOnlyRebuild = false;\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n    const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n    // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n    // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n    // edit, an embed model change -- still takes the full clear/reopen below.\n    if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n      console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n      db.exec('DROP TABLE content');\n      tokenizeOnlyRebuild = true;\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n  }\n\n  // Meta can lie after a crash between table creation and the signature write; the table's\n  // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n  // just dropped content above, so storedTokenize sees no table and this guard is skipped\n  // naturally rather than needing its own case.\n  const stored = storedTokenize(db);\n  if (stored !== null && stored !== tokenize) {\n    console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg, tokenize);\n\n  let rebuildWarnings: string[] = [];\n  if (tokenizeOnlyRebuild) {\n    rebuildWarnings = rebuildContentTable(db, cfg, cfg.baseDir);\n    setMeta(db, 'features', wantFeatures);\n  }\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","SenseError","activeFeatures","registerFunctions","changedSignatureKeys","rebuildContentTable","reconcile","signatureDiff","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","ensureSchema","tokenize","feature","schema","docCount","n","open","stateDir","baseDir","recursive","dbPath","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","clearCache","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AAClF,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AACtD,SAASC,iBAAiB,QAAQ,sBAAsB;AACxD,SAASC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AACrG,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAUnC,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,MAAMC,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAalB,gBAAgBiB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIpB,WAAW,kBAAkB,CAAC,kBAAkB,EAAEe,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAASG,aAAahB,EAAgB,EAAEC,GAAW,EAAEgB,QAAgB;IACnEjB,GAAGM,IAAI,CAAC,CAAC,oIAAoI,CAAC;IAC9I,wFAAwF;IACxF,4FAA4F;IAC5F,4FAA4F;IAC5F,2FAA2F;IAC3F,2FAA2F;IAC3F,2EAA2E;IAC3EN,GAAGM,IAAI,CAAC,CAAC,0IAA0I,EAAEW,SAAS,EAAE,CAAC;IACjK,6FAA6F;IAC7F,qFAAqF;IACrFjB,GAAGM,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1GN,GAAGM,IAAI,CAAC;IACR,KAAK,MAAMY,WAAW9B,eAAea,KAAMiB,QAAQC,MAAM,CAACnB;IAC1D,IAAIN,QAAQM,IAAI,sBAAsB,MAAML,QAAQK,IAAI,kBAAkBH;IAC1E,IAAIH,QAAQM,IAAI,gBAAgB,MAAML,QAAQK,IAAI,YAAYf,iBAAiBgB;AACjF;AAEA,OAAO,SAASmB,SAASpB,EAAgB;IACvC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,yCAAyCC,GAAG;IACnE,OAAOF,IAAIW,CAAC;AACd;AAEA,OAAO,SAASC,KAAKrB,GAAmB;QAwETP;IAvE7B,MAAM6B,WAAWzC,KAAKmB,IAAIuB,OAAO,EAAEtC;IACnCN,UAAU2C,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS5C,KAAKyC,UAAU3B;IAE9B,MAAMI,KAAK,IAAIjB,aAAa2C;IAC5B1B,GAAGM,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvCN,GAAGM,IAAI,CAAC;IACRjB,kBAAkBW,IAAIhB,gBAAgBiB,SAASE;IAE/CH,GAAGM,IAAI,CAAC;IAER,2FAA2F;IAC3F,8FAA8F;IAC9F,MAAMW,WAAWlB,gBAAgBC,IAAIC;IAErC,uFAAuF;IACvF,qGAAqG;IACrG,MAAM0B,UAAUjC,QAAQM,IAAI;IAC5B,MAAM4B,WAAWlC,QAAQM,IAAI;IAC7B,MAAM6B,eAAe5C,iBAAiBgB;IACtC,IAAI6B,sBAAsB;IAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY9B,kBAAoB+B,aAAa,QAAQA,aAAaC,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAIF,YAAY,QAAQA,YAAY9B,gBAAgB;YAClDkC,QAAQC,KAAK,CAAC;YACdhC,GAAGiC,KAAK;YACRC,WAAWjC;YACX,OAAOqB,KAAKrB;QACd;QACA,MAAMkC,cAAc7C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC;QACzD,0FAA0F;QAC1F,yFAAyF;QACzF,0EAA0E;QAC1E,IAAIM,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;YACzDN,QAAQC,KAAK,CAAC;YACdhC,GAAGM,IAAI,CAAC;YACRwB,sBAAsB;QACxB,OAAO;YACL,MAAMQ,UAAU7C,cAAcmC,qBAAAA,sBAAAA,WAAY,IAAIC;YAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEM,QAAQ,oBAAoB,CAAC;YACpEtC,GAAGiC,KAAK;YACRC,WAAWjC;YACX,OAAOqB,KAAKrB;QACd;IACF;IAEA,yFAAyF;IACzF,6FAA6F;IAC7F,wFAAwF;IACxF,8CAA8C;IAC9C,MAAMsC,SAAS9B,eAAeT;IAC9B,IAAIuC,WAAW,QAAQA,WAAWtB,UAAU;QAC1Cc,QAAQC,KAAK,CAAC;QACdhC,GAAGiC,KAAK;QACRC,WAAWjC;QACX,OAAOqB,KAAKrB;IACd;IAEAe,aAAahB,IAAIC,KAAKgB;IAEtB,IAAIuB,kBAA4B,EAAE;IAClC,IAAIV,qBAAqB;QACvBU,kBAAkBjD,oBAAoBS,IAAIC,KAAKA,IAAIuB,OAAO;QAC1D7B,QAAQK,IAAI,YAAY6B;IAC1B;IAEA,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMY,gBAAgBC,QAAOhD,WAAAA,QAAQM,IAAI,iCAAZN,sBAAAA,WAAmC;IAChEM,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;IAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAGvD,UAAUQ,IAAIC,KAAKA,IAAIuB,OAAO;IAE3D,OAAO;QAAExB;QAAIC;QAAKyB;QAAQoB;QAAQC,UAAU;eAAIP;eAAoBO;SAAS;IAAC;AAChF;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAASb,WAAWjC,GAAmB;IAC5CpB,OAAOC,KAAKmB,IAAIuB,OAAO,EAAEtC,YAAY;QAAEuC,WAAW;QAAMuB,OAAO;IAAK;AACtE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/open.ts"],"sourcesContent":["// The Node floor (>=22.20) is explained here and nowhere else: 22.20 is the first release with\n// both FTS5 and row-returning INSERT ... RETURNING. Raise it only for a load-bearing capability.\nimport { mkdirSync, rmSync } from 'node:fs';\nimport { join } from 'node:path';\nimport { DatabaseSync } from 'node:sqlite';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { contentTokenize, featureSignature, STATE_DIR } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport { registerFunctions } from '../sql-functions.ts';\nimport { changedSignatureKeys, embedIdentityAdopted, rebuildContentTable, reconcile, signatureDiff } from './reconcile.ts';\nimport { getMeta, setMeta } from './shared.ts';\n\nexport const DB_FILENAME = 'cache.db';\n// Cache shape version, independent of the config's own `version`. Bumping it rebuilds\n// existing trees on first query.\nexport const SCHEMA_VERSION = '17';\n\nexport interface OpenResult {\n  db: DatabaseSync;\n  cfg: ResolvedConfig;\n  dbPath: string;\n  parsed: number;\n  warnings: string[];\n}\n\n// Stemming is English-only, but the segmentation underneath it is what decides coverage:\n// unicode61 splits on spaces, so a language written without them (Chinese, Japanese, Thai)\n// indexes a whole run as one token and word search finds nothing. `content.tokenize` is how\n// such a tree picks trigram instead.\nconst DEFAULT_TOKENIZE = 'porter unicode61';\n\n// FTS5 takes its tokenizer as a string literal inside DDL, where nothing can bind, so a\n// configured value has to be concatenated. Probing a throwaway table is what makes that safe\n// and is also the whole validation: anything the linked SQLite accepts passes, anything else\n// fails here with SQLite's own message rather than against the real table. It means no table\n// of which version added which tokenizer has to be maintained.\nfunction resolveTokenize(db: DatabaseSync, cfg: Config): string {\n  const configured = contentTokenize(cfg);\n  if (configured === undefined) return DEFAULT_TOKENIZE;\n  const literal = configured.replace(/'/g, \"''\");\n  try {\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n    db.exec(`CREATE VIRTUAL TABLE temp.sense_tokenize_probe USING fts5(x, tokenize = '${literal}')`);\n    db.exec('DROP TABLE IF EXISTS temp.sense_tokenize_probe');\n  } catch (err) {\n    throw new SenseError('CONFIG_INVALID', `content.tokenize \"${configured}\" is not a tokenizer this SQLite accepts (${(err as Error).message}); the built-in choices are unicode61, ascii, porter, and trigram, each with their own options`);\n  }\n  return literal;\n}\n\n// The tokenizer the content table was actually built with, from its own DDL -- the one\n// record that cannot desynchronize from the table. NULL when the table does not exist yet.\nfunction storedTokenize(db: DatabaseSync): string | null {\n  const row = db.prepare(`SELECT sql FROM sqlite_master WHERE name = 'content'`).get() as { sql: string } | undefined;\n  if (!row) return null;\n  const m = row.sql.match(/tokenize = '((?:[^']|'')*)'/);\n  return m ? m[1] : null;\n}\n\n// Content is a separate table (not a column on frontmatter) so `SELECT * FROM frontmatter`\n// can't dump file text into context. Features add their own tables after the core ones.\nfunction ensureSchema(db: DatabaseSync, cfg: Config, tokenize: string): void {\n  db.exec(`CREATE TABLE IF NOT EXISTS frontmatter (\"path\" TEXT PRIMARY KEY, \"_mtime\" REAL, \"_ctime\" REAL, \"_size\" INTEGER, \"_parse_error\" TEXT)`);\n  // IF NOT EXISTS is safe against a tokenizer change: open() compares the table's own DDL\n  // against the resolved tokenizer before this runs, so a stale table is already gone by now.\n  // The three `_seg` sidecars are appended after path, never inserted: bm25(content, ...) and\n  // snippet(content, 2, ...) are documented against the first three columns and keep working\n  // (FTS5 defaults the weights it was not given). Each carries its field's exploded unspaced\n  // runs for text that needs it, and an empty string for text that does not.\n  db.exec(`CREATE VIRTUAL TABLE IF NOT EXISTS content USING fts5(title, summary, text, path UNINDEXED, title_seg, summary_seg, text_seg, tokenize = '${tokenize}')`);\n  // Coverage, not ownership: a path can appear under several presets. path leads the PK so the\n  // per-doc delete is an index hit -- keyed the other way, cold builds went quadratic.\n  db.exec(`CREATE TABLE IF NOT EXISTS preset_files (\"path\" TEXT, preset TEXT, PRIMARY KEY (\"path\", preset))`);\n  db.exec('CREATE INDEX IF NOT EXISTS preset_files_preset ON preset_files(preset)');\n  for (const feature of activeFeatures(cfg)) feature.schema(db);\n  if (getMeta(db, 'schema_version') === null) setMeta(db, 'schema_version', SCHEMA_VERSION);\n  if (getMeta(db, 'features') === null) setMeta(db, 'features', featureSignature(cfg));\n}\n\nexport function docCount(db: DatabaseSync): number {\n  const row = db.prepare('SELECT COUNT(*) AS n FROM frontmatter').get() as { n: number };\n  return row.n;\n}\n\nexport function open(cfg: ResolvedConfig): OpenResult {\n  const stateDir = join(cfg.baseDir, STATE_DIR);\n  mkdirSync(stateDir, { recursive: true });\n  const dbPath = join(stateDir, DB_FILENAME);\n\n  const db = new DatabaseSync(dbPath);\n  db.exec('PRAGMA journal_mode = WAL');\n  // Covers a concurrent watcher's bulk reconcile (~5s for 500 files at 26k notes). A query\n  // that outwaits it still fails loudly.\n  db.exec('PRAGMA busy_timeout = 30000');\n  registerFunctions(db, contentTokenize(cfg) === undefined);\n\n  db.exec('CREATE TABLE IF NOT EXISTS meta (key TEXT PRIMARY KEY, value TEXT)');\n\n  // Before the rebuild branch below, never after: that branch deletes the cache, so a typo'd\n  // tokenizer validated later would cost a full re-index (and re-embed) to reach its own error.\n  const tokenize = resolveTokenize(db, cfg);\n\n  // Schema-version or feature-set mismatch: reconcile only reparses changed files, so an\n  // old cache can't be patched incrementally -- rebuild instead (cheap: nothing expensive lives here).\n  const version = getMeta(db, 'schema_version');\n  const features = getMeta(db, 'features');\n  const wantFeatures = featureSignature(cfg);\n  let tokenizeOnlyRebuild = false;\n  if ((version !== null && version !== SCHEMA_VERSION) || (features !== null && features !== wantFeatures)) {\n    // Indexing derives from presets, so a config edit rebuilding the cache must say so and\n    // name what changed -- silent rebuilds make derived indexing look like a hang or a bug.\n    if (version !== null && version !== SCHEMA_VERSION) {\n      console.error('sense: cache format changed (new sensemaking version); rebuilding the index');\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n    const changedKeys = changedSignatureKeys(features ?? '', wantFeatures);\n    // Only the tokenizer moved: frontmatter, links, sections, and embeddings are file-derived\n    // and tokenizer-independent, so they don't need re-deriving. Everything else -- a preset\n    // edit, an embed model change -- still takes the full clear/reopen below.\n    if (changedKeys.size === 1 && changedKeys.has('tokenize')) {\n      console.error('sense: config change (content tokenizer) rebuilds the text index; vectors, links, and sections are kept');\n      db.exec('DROP TABLE content');\n      tokenizeOnlyRebuild = true;\n    } else if (changedKeys.size === 1 && changedKeys.has('embed') && embedIdentityAdopted(features ?? '', wantFeatures)) {\n      // First sight of a resolved weight identity: the model itself hasn't changed, so\n      // adopt it into meta with no rebuild and no re-embed, mirroring the tokenize precedent.\n      console.error(\"sense: recorded the embedding model's resolved identity; vectors are unaffected\");\n      setMeta(db, 'features', wantFeatures);\n    } else {\n      const changed = signatureDiff(features ?? '', wantFeatures);\n      console.error(`sense: config change (${changed}) rebuilds the index`);\n      db.close();\n      clearCache(cfg);\n      return open(cfg);\n    }\n  }\n\n  // Meta can lie after a crash between table creation and the signature write; the table's\n  // own DDL cannot. A mismatch here rebuilds no matter what meta says. A tokenize-only rebuild\n  // just dropped content above, so storedTokenize sees no table and this guard is skipped\n  // naturally rather than needing its own case.\n  const stored = storedTokenize(db);\n  if (stored !== null && stored !== tokenize) {\n    console.error('sense: cache was built with a different content tokenizer; rebuilding the index');\n    db.close();\n    clearCache(cfg);\n    return open(cfg);\n  }\n\n  ensureSchema(db, cfg, tokenize);\n\n  let rebuildWarnings: string[] = [];\n  if (tokenizeOnlyRebuild) {\n    rebuildWarnings = rebuildContentTable(db, cfg, cfg.baseDir);\n    setMeta(db, 'features', wantFeatures);\n  }\n\n  // 3x the largest reconcile this cache has recorded, floored at 30s and capped at 10min.\n  // Installed before reconcile() -- that call is the one that races a watcher's transaction.\n  const recordedMaxMs = Number(getMeta(db, 'reconcile_max_ms') ?? '0');\n  db.exec(`PRAGMA busy_timeout = ${Math.min(Math.max(30000, 3 * recordedMaxMs), 600_000)}`);\n\n  const { parsed, warnings } = reconcile(db, cfg, cfg.baseDir);\n\n  return { db, cfg, dbPath, parsed, warnings: [...rebuildWarnings, ...warnings] };\n}\n\n// Deletes the cache directory, and only that. The rebuild is not this function's job: the next\n// open() reconciles, which is what running any command already does, so a verb that bundled the\n// two described the half that was not its own. Manual reset for a doubted cache; the schema and\n// config-signature mismatches below reset themselves.\nexport function clearCache(cfg: ResolvedConfig): void {\n  rmSync(join(cfg.baseDir, STATE_DIR), { recursive: true, force: true });\n}\n"],"names":["mkdirSync","rmSync","join","DatabaseSync","contentTokenize","featureSignature","STATE_DIR","SenseError","activeFeatures","registerFunctions","changedSignatureKeys","embedIdentityAdopted","rebuildContentTable","reconcile","signatureDiff","getMeta","setMeta","DB_FILENAME","SCHEMA_VERSION","DEFAULT_TOKENIZE","resolveTokenize","db","cfg","configured","undefined","literal","replace","exec","err","message","storedTokenize","row","prepare","get","m","sql","match","ensureSchema","tokenize","feature","schema","docCount","n","open","stateDir","baseDir","recursive","dbPath","version","features","wantFeatures","tokenizeOnlyRebuild","console","error","close","clearCache","changedKeys","size","has","changed","stored","rebuildWarnings","recordedMaxMs","Number","Math","min","max","parsed","warnings","force"],"mappings":"AAAA,+FAA+F;AAC/F,iGAAiG;AACjG,SAASA,SAAS,EAAEC,MAAM,QAAQ,UAAU;AAC5C,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,YAAY,QAAQ,cAAc;AAE3C,SAASC,eAAe,EAAEC,gBAAgB,EAAEC,SAAS,QAAQ,qBAAqB;AAClF,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AACtD,SAASC,iBAAiB,QAAQ,sBAAsB;AACxD,SAASC,oBAAoB,EAAEC,oBAAoB,EAAEC,mBAAmB,EAAEC,SAAS,EAAEC,aAAa,QAAQ,iBAAiB;AAC3H,SAASC,OAAO,EAAEC,OAAO,QAAQ,cAAc;AAE/C,OAAO,MAAMC,cAAc,WAAW;AACtC,sFAAsF;AACtF,iCAAiC;AACjC,OAAO,MAAMC,iBAAiB,KAAK;AAUnC,yFAAyF;AACzF,2FAA2F;AAC3F,4FAA4F;AAC5F,qCAAqC;AACrC,MAAMC,mBAAmB;AAEzB,wFAAwF;AACxF,6FAA6F;AAC7F,6FAA6F;AAC7F,6FAA6F;AAC7F,+DAA+D;AAC/D,SAASC,gBAAgBC,EAAgB,EAAEC,GAAW;IACpD,MAAMC,aAAanB,gBAAgBkB;IACnC,IAAIC,eAAeC,WAAW,OAAOL;IACrC,MAAMM,UAAUF,WAAWG,OAAO,CAAC,MAAM;IACzC,IAAI;QACFL,GAAGM,IAAI,CAAC;QACRN,GAAGM,IAAI,CAAC,CAAC,yEAAyE,EAAEF,QAAQ,EAAE,CAAC;QAC/FJ,GAAGM,IAAI,CAAC;IACV,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAIrB,WAAW,kBAAkB,CAAC,kBAAkB,EAAEgB,WAAW,0CAA0C,EAAE,AAACK,IAAcC,OAAO,CAAC,8FAA8F,CAAC;IAC3O;IACA,OAAOJ;AACT;AAEA,uFAAuF;AACvF,2FAA2F;AAC3F,SAASK,eAAeT,EAAgB;IACtC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,CAAC,oDAAoD,CAAC,EAAEC,GAAG;IAClF,IAAI,CAACF,KAAK,OAAO;IACjB,MAAMG,IAAIH,IAAII,GAAG,CAACC,KAAK,CAAC;IACxB,OAAOF,IAAIA,CAAC,CAAC,EAAE,GAAG;AACpB;AAEA,2FAA2F;AAC3F,wFAAwF;AACxF,SAASG,aAAahB,EAAgB,EAAEC,GAAW,EAAEgB,QAAgB;IACnEjB,GAAGM,IAAI,CAAC,CAAC,oIAAoI,CAAC;IAC9I,wFAAwF;IACxF,4FAA4F;IAC5F,4FAA4F;IAC5F,2FAA2F;IAC3F,2FAA2F;IAC3F,2EAA2E;IAC3EN,GAAGM,IAAI,CAAC,CAAC,0IAA0I,EAAEW,SAAS,EAAE,CAAC;IACjK,6FAA6F;IAC7F,qFAAqF;IACrFjB,GAAGM,IAAI,CAAC,CAAC,gGAAgG,CAAC;IAC1GN,GAAGM,IAAI,CAAC;IACR,KAAK,MAAMY,WAAW/B,eAAec,KAAMiB,QAAQC,MAAM,CAACnB;IAC1D,IAAIN,QAAQM,IAAI,sBAAsB,MAAML,QAAQK,IAAI,kBAAkBH;IAC1E,IAAIH,QAAQM,IAAI,gBAAgB,MAAML,QAAQK,IAAI,YAAYhB,iBAAiBiB;AACjF;AAEA,OAAO,SAASmB,SAASpB,EAAgB;IACvC,MAAMU,MAAMV,GAAGW,OAAO,CAAC,yCAAyCC,GAAG;IACnE,OAAOF,IAAIW,CAAC;AACd;AAEA,OAAO,SAASC,KAAKrB,GAAmB;QA6ETP;IA5E7B,MAAM6B,WAAW1C,KAAKoB,IAAIuB,OAAO,EAAEvC;IACnCN,UAAU4C,UAAU;QAAEE,WAAW;IAAK;IACtC,MAAMC,SAAS7C,KAAK0C,UAAU3B;IAE9B,MAAMI,KAAK,IAAIlB,aAAa4C;IAC5B1B,GAAGM,IAAI,CAAC;IACR,yFAAyF;IACzF,uCAAuC;IACvCN,GAAGM,IAAI,CAAC;IACRlB,kBAAkBY,IAAIjB,gBAAgBkB,SAASE;IAE/CH,GAAGM,IAAI,CAAC;IAER,2FAA2F;IAC3F,8FAA8F;IAC9F,MAAMW,WAAWlB,gBAAgBC,IAAIC;IAErC,uFAAuF;IACvF,qGAAqG;IACrG,MAAM0B,UAAUjC,QAAQM,IAAI;IAC5B,MAAM4B,WAAWlC,QAAQM,IAAI;IAC7B,MAAM6B,eAAe7C,iBAAiBiB;IACtC,IAAI6B,sBAAsB;IAC1B,IAAI,AAACH,YAAY,QAAQA,YAAY9B,kBAAoB+B,aAAa,QAAQA,aAAaC,cAAe;QACxG,uFAAuF;QACvF,wFAAwF;QACxF,IAAIF,YAAY,QAAQA,YAAY9B,gBAAgB;YAClDkC,QAAQC,KAAK,CAAC;YACdhC,GAAGiC,KAAK;YACRC,WAAWjC;YACX,OAAOqB,KAAKrB;QACd;QACA,MAAMkC,cAAc9C,qBAAqBuC,qBAAAA,sBAAAA,WAAY,IAAIC;QACzD,0FAA0F;QAC1F,yFAAyF;QACzF,0EAA0E;QAC1E,IAAIM,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,aAAa;YACzDN,QAAQC,KAAK,CAAC;YACdhC,GAAGM,IAAI,CAAC;YACRwB,sBAAsB;QACxB,OAAO,IAAIK,YAAYC,IAAI,KAAK,KAAKD,YAAYE,GAAG,CAAC,YAAY/C,qBAAqBsC,qBAAAA,sBAAAA,WAAY,IAAIC,eAAe;YACnH,iFAAiF;YACjF,wFAAwF;YACxFE,QAAQC,KAAK,CAAC;YACdrC,QAAQK,IAAI,YAAY6B;QAC1B,OAAO;YACL,MAAMS,UAAU7C,cAAcmC,qBAAAA,sBAAAA,WAAY,IAAIC;YAC9CE,QAAQC,KAAK,CAAC,CAAC,sBAAsB,EAAEM,QAAQ,oBAAoB,CAAC;YACpEtC,GAAGiC,KAAK;YACRC,WAAWjC;YACX,OAAOqB,KAAKrB;QACd;IACF;IAEA,yFAAyF;IACzF,6FAA6F;IAC7F,wFAAwF;IACxF,8CAA8C;IAC9C,MAAMsC,SAAS9B,eAAeT;IAC9B,IAAIuC,WAAW,QAAQA,WAAWtB,UAAU;QAC1Cc,QAAQC,KAAK,CAAC;QACdhC,GAAGiC,KAAK;QACRC,WAAWjC;QACX,OAAOqB,KAAKrB;IACd;IAEAe,aAAahB,IAAIC,KAAKgB;IAEtB,IAAIuB,kBAA4B,EAAE;IAClC,IAAIV,qBAAqB;QACvBU,kBAAkBjD,oBAAoBS,IAAIC,KAAKA,IAAIuB,OAAO;QAC1D7B,QAAQK,IAAI,YAAY6B;IAC1B;IAEA,wFAAwF;IACxF,2FAA2F;IAC3F,MAAMY,gBAAgBC,QAAOhD,WAAAA,QAAQM,IAAI,iCAAZN,sBAAAA,WAAmC;IAChEM,GAAGM,IAAI,CAAC,CAAC,sBAAsB,EAAEqC,KAAKC,GAAG,CAACD,KAAKE,GAAG,CAAC,OAAO,IAAIJ,gBAAgB,SAAU;IAExF,MAAM,EAAEK,MAAM,EAAEC,QAAQ,EAAE,GAAGvD,UAAUQ,IAAIC,KAAKA,IAAIuB,OAAO;IAE3D,OAAO;QAAExB;QAAIC;QAAKyB;QAAQoB;QAAQC,UAAU;eAAIP;eAAoBO;SAAS;IAAC;AAChF;AAEA,+FAA+F;AAC/F,gGAAgG;AAChG,gGAAgG;AAChG,sDAAsD;AACtD,OAAO,SAASb,WAAWjC,GAAmB;IAC5CrB,OAAOC,KAAKoB,IAAIuB,OAAO,EAAEvC,YAAY;QAAEwC,WAAW;QAAMuB,OAAO;IAAK;AACtE"}

package/dist/esm/db/reconcile.d.ts

@@ -5,5 +5,6 @@ export declare function reconcile(db: DatabaseSync, cfg: Config, baseDir: string
     warnings: string[];
 };
 export declare function changedSignatureKeys(before: string, after: string): Set<string>;
+export declare function embedIdentityAdopted(before: string, after: string): boolean;
 export declare function signatureDiff(before: string, after: string): string;
 export declare function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[];

package/dist/esm/db/reconcile.js

@@ -178,6 +178,16 @@ export function changedSignatureKeys(before, after) {
     for (const key of a.keys())if (!b.has(key)) changed.add(key);
     return changed;
 }
+// Whether the embed segment only gained its resolved weight identity: same provider and
+// model, no identity recorded before, one now -- adopted into meta without a rebuild.
+export function embedIdentityAdopted(before, after) {
+    const embedPart = (sig)=>sig.split('|').find((p)=>p.startsWith('embed:'));
+    const b = embedPart(before);
+    const a = embedPart(after);
+    if (b === undefined || a === undefined) return false;
+    const at = a.indexOf('@');
+    return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;
+}
 // Names what moved, for the rebuild notice.
 export function signatureDiff(before, after) {
     const changed = changedSignatureKeys(before, after);

package/dist/esm/db/reconcile.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/reconcile.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport type { Config } from '../config/index.ts';\nimport { contentTokenize } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../progress.ts';\nimport type { ParsedDoc } from '../scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../scan.ts';\nimport { segmentField } from '../segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from './shared.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","getColumns","getMeta","quoteIdent","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","segmenting","run","relPath","search","title","summary","text","reconcile","db","cfg","baseDir","files","currentSet","map","f","existingRows","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","feature","enabledForFile","fileWarnings","tick","push","key","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","txStart","Date","now","exec","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","undefined","values","ctimeMs","parseError","presetName","presets","store","extracted","name","afterReconcile","err","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","val","signatureDiff","label","rebuildContentTable","known"],"mappings":"AAEA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,iBAAiB;AAE1C,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,aAAa;AACpE,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,cAAc;AAEvE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEC,UAAmB;IACtFF,WAAWG,GAAG,CAACF,IAAIG,OAAO,EAAEH,IAAII,MAAM,CAACC,KAAK,EAAEL,IAAII,MAAM,CAACE,OAAO,EAAEN,IAAII,MAAM,CAACG,IAAI,EAAEP,IAAIG,OAAO,EAAEF,aAAaZ,aAAaW,IAAII,MAAM,CAACC,KAAK,IAAI,IAAIJ,aAAaZ,aAAaW,IAAII,MAAM,CAACE,OAAO,IAAI,IAAIL,aAAaZ,aAAaW,IAAII,MAAM,CAACG,IAAI,IAAI;AACrP;AAEA,OAAO,SAASC,UAAUC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIlB,IAAIiB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAeP,GAAGQ,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMC,WAAW,IAAIC,IAAIJ,aAAaF,GAAG,CAAC,CAACO,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWP,aAAaQ,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGR,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYd,MAAMY,MAAM,CAAC,CAACT;QAC9B,MAAMY,MAAMR,SAASS,GAAG,CAACb,EAAEZ,OAAO;QAClC,OAAO,CAACwB,OAAOA,IAAIE,MAAM,KAAKd,EAAEe,OAAO,IAAIH,IAAII,KAAK,KAAKhB,EAAEiB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWpD,eAAe0B;IAChC,MAAM2B,cAAc/C,WAAWmB;IAC/B,MAAM6B,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASvD,SAAS,mBAAmByC,UAAUO,MAAM;IAC3D,IAAIQ,cAAc;IAClB,KAAK,MAAMC,QAAQhB,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMiB,eAAeP,SAASZ,MAAM,CAAC,CAACoB,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACnC,KAAKgC;QACzG,MAAM,EAAE1C,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMG,OAAOC,OAAOC,IAAI,CAACnD,IAAIoD,IAAI,EAAG;YACvC,IAAI,CAACf,YAAYZ,GAAG,CAACwB,MAAM;gBACzBZ,YAAYgB,GAAG,CAACJ;gBAChBX,WAAWU,IAAI,CAACC;YAClB;QACF;QACAV,WAAWS,IAAI,CAAChD;IAClB;IACAwC,OAAOc,MAAM;IAEb,MAAMC,aAAa;WAAIlB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIkB,WAAWtB,MAAM,GAAGrC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAEwE,WAAWtB,MAAM,CAAC,0EAA0E,EAAErC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAM4D,kBAAkBD,WAAW/B,MAAM,CAAC,CAACiC,IAAM/D,yBAAyB+B,GAAG,CAACgC,MAAM,CAACrE,iBAAiBqC,GAAG,CAACgC;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB1C,GAAG,CAACtB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB1C,GAAG,CAAC,IAAM,KAAK6C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLhC,MAAM,CAAC,CAACiC,IAAMA,MAAM,QACpB3C,GAAG,CAAC,CAAC2C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQlC,UAAUF,MAAM,CAAC,CAACT,IAAM,CAACI,SAASM,GAAG,CAACV,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAM0D,QAAwB;QAAEjD;QAAOkD,UAAUvB,WAAWzB,GAAG,CAAC,CAACiD,IAAMA,EAAE5D,OAAO;QAAGyD;QAAOrC;IAAS;IAEnG,MAAMyC,UAAUC,KAAKC,GAAG;IACxBzD,GAAG0D,IAAI,CAAC;IACR,IAAI;YAkD8BvB;QAjDhC,KAAK,MAAMwB,OAAO9B,WAAY7B,GAAG0D,IAAI,CAAC,CAAC,mCAAmC,EAAE3E,WAAW4E,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5D,GAAGQ,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMqD,iBAAiB7D,GAAGQ,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMsD,mBAAmB9D,GAAGQ,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAIM,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMuC,MAAM/D,GAAGQ,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMK,QAAQC,SAAU;oBAMKqB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEyB,QAAQnE,GAAG,CAACoB;gBACZkD,IAAItE,GAAG,CAACoB;gBACRgD,eAAepE,GAAG,CAACoB;gBACnB,KAAK,MAAMsB,WAAWR,UAAUQ,kBAAAA,QAAQ6B,MAAM,cAAd7B,sCAAAA,qBAAAA,SAAiBnC,IAAIa,MAAMuC;YAC7D;QACF;QACA,IAAItB,WAAWN,MAAM,GAAG,GAAG;YACzB,MAAMyC,SAASjE,GAAGQ,OAAO,CAACyC;YAC1B,MAAM3D,aAAaU,GAAGQ,OAAO,CAACpB;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMI,aAAanB,gBAAgB4B,SAASiE;YAC5C,KAAK,MAAM3E,OAAOuC,WAAY;oBAsBIK;gBArBhC,MAAMgC,SAASpB,gBAAgB1C,GAAG,CAAC,CAACsD;wBAO3BpE;oBANP,IAAIoE,QAAQ,QAAQ,OAAOpE,IAAIG,OAAO;oBACtC,IAAIiE,QAAQ,UAAU,OAAOpE,IAAI8B,OAAO;oBACxC,IAAIsC,QAAQ,UAAU,OAAOpE,IAAI6E,OAAO;oBACxC,IAAIT,QAAQ,SAAS,OAAOpE,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIoC,QAAQ,gBAAgB,OAAOpE,IAAI8E,UAAU;oBACjD,QAAO9E,gBAAAA,IAAIoD,IAAI,CAACgB,IAAI,cAAbpE,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0E,OAAOxE,GAAG,IAAI0E;gBACd,oFAAoF;gBACpF,IAAIzD,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAG;wBAEGyC;oBADhCyB,QAAQnE,GAAG,CAACF,IAAIG,OAAO;oBACvB,KAAK,MAAMyC,WAAWR,UAAUQ,mBAAAA,QAAQ6B,MAAM,cAAd7B,uCAAAA,sBAAAA,SAAiBnC,IAAIT,IAAIG,OAAO,EAAE0D;gBACpE;gBACA/D,iBAAiBC,YAAYC,KAAKC;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIkB,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAGmE,eAAepE,GAAG,CAACF,IAAIG,OAAO;gBAC7D,KAAK,MAAM4E,cAAc/E,IAAIgF,OAAO,CAAET,iBAAiBrE,GAAG,CAACF,IAAIG,OAAO,EAAE4E;gBACxE,KAAK,MAAMnC,WAAWR,UAAUQ,iBAAAA,QAAQqC,KAAK,cAAbrC,qCAAAA,oBAAAA,SAAgBnC,IAAIT,IAAIG,OAAO,EAAEH,IAAIkF,SAAS,CAACtC,QAAQuC,IAAI,CAAC,EAAEtB;YAChG;QACF;QACA,KAAK,MAAMjB,WAAWR,UAAUQ,0BAAAA,QAAQwC,cAAc,cAAtBxC,8CAAAA,6BAAAA,SAAyBnC,IAAIoD;QAC7DpD,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAUhG,QAAQkB,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM+E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS/F,QAAQgB,IAAI,oBAAoBiF,OAAOJ;IAEjE,OAAO;QAAEpD,QAAQK,WAAWN,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASwD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGvC,IAAI,CAAC,OAAOoC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAIhF,IAAIgF,IAAIH,KAAK,CAAC,KAAKnF,GAAG,CAAC,CAACiF,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAI5G;IACpB,KAAK,MAAM,CAACsD,KAAKuD,IAAI,IAAIF,EAAG,IAAID,EAAEzE,GAAG,CAACqB,SAASuD,KAAKD,QAAQlD,GAAG,CAACJ;IAChE,KAAK,MAAMA,OAAOoD,EAAElD,IAAI,GAAI,IAAI,CAACmD,EAAE7E,GAAG,CAACwB,MAAMsD,QAAQlD,GAAG,CAACJ;IACzD,OAAOsD;AACT;AAEA,4CAA4C;AAC5C,OAAO,SAASE,cAAcb,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMa,QAAQ,CAACzD,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAI+C,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAE/C,IAAIiD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQvE,IAAI,KAAK,IAAI,aAAa;WAAIuE;KAAQ,CAACzF,GAAG,CAAC4F,OAAO/C,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,SAASgD,oBAAoBlG,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAChF,MAAMiG,QAAQ,IAAIjH,IAAI,AAACc,GAAGQ,OAAO,CAAC,kCAAkCC,GAAG,GAA+BJ,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IACvH,MAAMV,QAAQ1B,UAAUwB,KAAKC,SAASa,MAAM,CAAC,CAACT,IAAM6F,MAAMnF,GAAG,CAACV,EAAEZ,OAAO;IACvE,MAAMF,aAAanB,gBAAgB4B,SAASiE;IAC5C,MAAM5E,aAAaU,GAAGQ,OAAO,CAACpB;IAC9B,MAAMsC,WAAqB,EAAE;IAC7B1B,GAAG0D,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMzB,QAAQ9B,MAAO;YACxB,MAAM,EAAEZ,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD;YAClDP,SAASa,IAAI,IAAIF;YACjBhD,iBAAiBC,YAAYC,KAAKC;QACpC;QACAQ,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IACA,OAAOlD;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/db/reconcile.ts"],"sourcesContent":["import type { DatabaseSync, StatementSync } from 'node:sqlite';\nimport type { Config } from '../config/index.ts';\nimport { contentTokenize } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { activeFeatures } from '../features/index.ts';\nimport type { ReconcileDelta } from '../features/types.ts';\nimport { progress } from '../progress.ts';\nimport type { ParsedDoc } from '../scan.ts';\nimport { listFiles, parseFile, RESERVED_COLUMNS } from '../scan.ts';\nimport { segmentField } from '../segment.ts';\nimport { getColumns, getMeta, quoteIdent, setMeta } from './shared.ts';\n\n// Feature-owned columns (`_rank`) must stay out of the upsert: a reparse would null the last\n// computed value on every touch, not just the reconciles that recompute it.\nconst CORE_FRONTMATTER_COLUMNS = new Set(['path', '_mtime', '_ctime', '_size', '_parse_error']);\n\n// SQLite's compile-time SQLITE_MAX_COLUMN, default 2000 (https://www.sqlite.org/limits.html).\nconst MAX_FRONTMATTER_COLUMNS = 2000;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so both prepare the same\n// literal instead of two copies drifting apart.\nconst INSERT_CONTENT_SQL = `INSERT INTO content (rowid, title, summary, text, \"path\", title_seg, summary_seg, text_seg) VALUES ((SELECT rowid FROM frontmatter WHERE \"path\" = ?), ?, ?, ?, ?, ?, ?, ?)`;\n\n// Shared by reconcile() and the tokenize-only rebuild in open(), so content population is\n// defined once. Assumes the frontmatter row for doc.relPath already exists (rowid lookup).\nfunction insertContentRow(insertBody: StatementSync, doc: ParsedDoc, segmenting: boolean): void {\n  insertBody.run(doc.relPath, doc.search.title, doc.search.summary, doc.search.text, doc.relPath, segmenting ? segmentField(doc.search.title) : '', segmenting ? segmentField(doc.search.summary) : '', segmenting ? segmentField(doc.search.text) : '');\n}\n\nexport function reconcile(db: DatabaseSync, cfg: Config, baseDir: string): { parsed: number; warnings: string[] } {\n  const files = listFiles(cfg, baseDir);\n  const currentSet = new Set(files.map((f) => f.relPath));\n\n  const existingRows = db.prepare(`SELECT \"path\", \"_mtime\", \"_size\" FROM frontmatter`).all() as Array<{\n    path: string;\n    _mtime: number;\n    _size: number;\n  }>;\n  const existing = new Map(existingRows.map((r) => [r.path, r]));\n  const vanished = existingRows.filter((r) => !currentSet.has(r.path)).map((r) => r.path);\n\n  const toReparse = files.filter((f) => {\n    const row = existing.get(f.relPath);\n    return !row || row._mtime !== f.mtimeMs || row._size !== f.size;\n  });\n\n  if (vanished.length === 0 && toReparse.length === 0) return { parsed: 0, warnings: [] };\n\n  const features = activeFeatures(cfg);\n  const seenColumns = getColumns(db);\n  const newColumns: string[] = [];\n  const parsedDocs: ParsedDoc[] = [];\n  const warnings: string[] = [];\n\n  // Bulk reparses (a sync, a cold build) are the long silences a query can hit; short\n  // reconciles stay silent (progress() has a threshold).\n  const report = progress('reparsing files', toReparse.length);\n  let parsedCount = 0;\n  for (const file of toReparse) {\n    // A doc only gets extract/store from features that apply to it (currently: embed, via\n    // FileStat.embed -- true iff the config names an embedding model).\n    const fileFeatures = features.filter((feature) => !feature.enabledForFile || feature.enabledForFile(cfg, file));\n    const { doc, warnings: fileWarnings } = parseFile(file, fileFeatures);\n    report.tick(++parsedCount);\n    warnings.push(...fileWarnings);\n    for (const key of Object.keys(doc.data)) {\n      if (!seenColumns.has(key)) {\n        seenColumns.add(key);\n        newColumns.push(key);\n      }\n    }\n    parsedDocs.push(doc);\n  }\n  report.finish();\n\n  const allColumns = [...seenColumns];\n  // Fence before ALTERing: SQLite's own failure past this point is a raw\n  // \"too many columns on sqlite_altertab_frontmatter\" with no indication of the boundary or the levers.\n  if (allColumns.length > MAX_FRONTMATTER_COLUMNS) {\n    throw new SenseError(\n      'COLUMN_LIMIT',\n      `frontmatter would need ${allColumns.length} columns, crossing SQLite's compile-time SQLITE_MAX_COLUMN limit (default ${MAX_FRONTMATTER_COLUMNS}; see https://www.sqlite.org/limits.html). Narrow the presets' include globs so fewer/other files are indexed, or fix whatever is generating unbounded frontmatter keys.`\n    );\n  }\n  // Columns the frontmatter upsert actually writes: core + parsed frontmatter keys, never a\n  // feature-owned reserved column (see CORE_FRONTMATTER_COLUMNS above).\n  const writableColumns = allColumns.filter((c) => CORE_FRONTMATTER_COLUMNS.has(c) || !RESERVED_COLUMNS.has(c));\n  // ON CONFLICT UPDATE (not OR REPLACE) keeps the row's rowid stable across reparses --\n  // content rows are coupled to that rowid below.\n  const insertSql = `INSERT INTO frontmatter (${writableColumns.map(quoteIdent).join(', ')}) VALUES (${writableColumns.map(() => '?').join(', ')}) ON CONFLICT(\"path\") DO UPDATE SET ${writableColumns\n    .filter((c) => c !== 'path')\n    .map((c) => `${quoteIdent(c)} = excluded.${quoteIdent(c)}`)\n    .join(', ')}`;\n\n  const added = toReparse.filter((f) => !existing.has(f.relPath)).map((f) => f.relPath);\n  const delta: ReconcileDelta = { files, reparsed: parsedDocs.map((d) => d.relPath), added, vanished };\n\n  const txStart = Date.now();\n  db.exec('BEGIN');\n  try {\n    for (const col of newColumns) db.exec(`ALTER TABLE frontmatter ADD COLUMN ${quoteIdent(col)}`);\n    // FTS5 has no upsert, so delete-before-insert into `content`; coupled to the\n    // frontmatter rowid (indexed via its PRIMARY KEY) instead of the UNINDEXED `path`\n    // column, which a per-row DELETE would otherwise scan the whole table to find.\n    const delBody = db.prepare(`DELETE FROM content WHERE rowid = (SELECT rowid FROM frontmatter WHERE \"path\" = ?)`);\n    const delPresetFiles = db.prepare(`DELETE FROM preset_files WHERE \"path\" = ?`);\n    const insertPresetFile = db.prepare(`INSERT INTO preset_files (\"path\", preset) VALUES (?, ?)`);\n    if (vanished.length > 0) {\n      const del = db.prepare(`DELETE FROM frontmatter WHERE \"path\" = ?`);\n      for (const path of vanished) {\n        // content delete must run first: it looks up the frontmatter rowid by path, which\n        // the frontmatter delete below would otherwise have already removed.\n        delBody.run(path);\n        del.run(path);\n        delPresetFiles.run(path);\n        for (const feature of features) feature.remove?.(db, path, delta);\n      }\n    }\n    if (parsedDocs.length > 0) {\n      const insert = db.prepare(insertSql);\n      const insertBody = db.prepare(INSERT_CONTENT_SQL);\n      // A non-default tokenizer means the tree has chosen its own scheme; a phrase query over\n      // grapheme runs would be nonsense against trigram, so the sidecars stay empty.\n      const segmenting = contentTokenize(cfg) === undefined;\n      for (const doc of parsedDocs) {\n        const values = writableColumns.map((col) => {\n          if (col === 'path') return doc.relPath;\n          if (col === '_mtime') return doc.mtimeMs;\n          if (col === '_ctime') return doc.ctimeMs;\n          if (col === '_size') return doc.size;\n          // Written per parse, unlike _rank, which a feature pass owns and the upsert skips.\n          if (col === '_parse_error') return doc.parseError;\n          return doc.data[col] ?? null;\n        });\n        // Frontmatter upsert first: content's rowid lookup below depends on this row existing.\n        insert.run(...values);\n        // Only for docs that have rows: an unconditional delete made cold crawls quadratic.\n        if (existing.has(doc.relPath)) {\n          delBody.run(doc.relPath);\n          for (const feature of features) feature.remove?.(db, doc.relPath, delta);\n        }\n        insertContentRow(insertBody, doc, segmenting);\n        // A preset edit forces a full rebuild, so an unchanged doc's coverage is already\n        // correct; new docs have nothing to clear, which keeps cold builds linear.\n        if (existing.has(doc.relPath)) delPresetFiles.run(doc.relPath);\n        for (const presetName of doc.presets) insertPresetFile.run(doc.relPath, presetName);\n        for (const feature of features) feature.store?.(db, doc.relPath, doc.extracted[feature.name], delta);\n      }\n    }\n    for (const feature of features) feature.afterReconcile?.(db, delta);\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n\n  // Reconcile's own write-transaction duration, for open()'s derived busy_timeout (F):\n  // keep the observed max so a big watcher reconcile's lock hold is what the next open bounds its wait against.\n  const durationMs = Date.now() - txStart;\n  const prevRaw = getMeta(db, 'reconcile_max_ms');\n  // -1, not 0, so a genuinely 0ms first reconcile (sub-millisecond, common on a tiny tree)\n  // still gets recorded instead of losing to the \"nothing recorded yet\" default.\n  const prevMax = prevRaw === null ? -1 : Number(prevRaw);\n  if (durationMs > prevMax) setMeta(db, 'reconcile_max_ms', String(durationMs));\n\n  return { parsed: parsedDocs.length, warnings };\n}\n\n// Segment keys that moved between two feature signatures (see config.featureSignature's\n// format: global features, embed provider, tokenize, then one segment per preset).\nexport function changedSignatureKeys(before: string, after: string): Set<string> {\n  const keyOf = (part: string) => (part.startsWith('preset:') ? part.split(':').slice(0, 2).join(':') : part.split(':')[0]);\n  const parse = (sig: string) => new Map(sig.split('|').map((part) => [keyOf(part), part]));\n  const a = parse(before);\n  const b = parse(after);\n  const changed = new Set<string>();\n  for (const [key, val] of b) if (a.get(key) !== val) changed.add(key);\n  for (const key of a.keys()) if (!b.has(key)) changed.add(key);\n  return changed;\n}\n\n// Whether the embed segment only gained its resolved weight identity: same provider and\n// model, no identity recorded before, one now -- adopted into meta without a rebuild.\nexport function embedIdentityAdopted(before: string, after: string): boolean {\n  const embedPart = (sig: string) => sig.split('|').find((p) => p.startsWith('embed:'));\n  const b = embedPart(before);\n  const a = embedPart(after);\n  if (b === undefined || a === undefined) return false;\n  const at = a.indexOf('@');\n  return b.indexOf('@') === -1 && at !== -1 && a.slice(0, at) === b;\n}\n\n// Names what moved, for the rebuild notice.\nexport function signatureDiff(before: string, after: string): string {\n  const changed = changedSignatureKeys(before, after);\n  const label = (key: string) => (key === 'embed' ? 'embed settings' : key === 'tokenize' ? 'content tokenizer' : key.startsWith('preset:') ? `preset \"${key.slice(7)}\"` : 'features');\n  return changed.size === 0 ? 'features' : [...changed].map(label).join(', ');\n}\n\n// The tokenize-only rebuild in open(): content is dropped and repopulated from files already\n// listed in frontmatter, which itself is untouched. Frontmatter, links, sections, and\n// embeddings are file-derived and tokenizer-independent, so they survive. No feature extractors\n// run here -- doc.search (title/summary/text) is all content population needs. Returns\n// parseFile's per-file warnings (e.g. a bad date) so open() can surface them -- mtimes are\n// untouched, so reconcile() never reparses these files and would otherwise never emit them again.\nexport function rebuildContentTable(db: DatabaseSync, cfg: Config, baseDir: string): string[] {\n  const known = new Set((db.prepare('SELECT \"path\" FROM frontmatter').all() as Array<{ path: string }>).map((r) => r.path));\n  const files = listFiles(cfg, baseDir).filter((f) => known.has(f.relPath));\n  const segmenting = contentTokenize(cfg) === undefined;\n  const insertBody = db.prepare(INSERT_CONTENT_SQL);\n  const warnings: string[] = [];\n  db.exec('BEGIN');\n  try {\n    for (const file of files) {\n      const { doc, warnings: fileWarnings } = parseFile(file);\n      warnings.push(...fileWarnings);\n      insertContentRow(insertBody, doc, segmenting);\n    }\n    db.exec('COMMIT');\n  } catch (err) {\n    db.exec('ROLLBACK');\n    throw err;\n  }\n  return warnings;\n}\n"],"names":["contentTokenize","SenseError","activeFeatures","progress","listFiles","parseFile","RESERVED_COLUMNS","segmentField","getColumns","getMeta","quoteIdent","setMeta","CORE_FRONTMATTER_COLUMNS","Set","MAX_FRONTMATTER_COLUMNS","INSERT_CONTENT_SQL","insertContentRow","insertBody","doc","segmenting","run","relPath","search","title","summary","text","reconcile","db","cfg","baseDir","files","currentSet","map","f","existingRows","prepare","all","existing","Map","r","path","vanished","filter","has","toReparse","row","get","_mtime","mtimeMs","_size","size","length","parsed","warnings","features","seenColumns","newColumns","parsedDocs","report","parsedCount","file","fileFeatures","feature","enabledForFile","fileWarnings","tick","push","key","Object","keys","data","add","finish","allColumns","writableColumns","c","insertSql","join","added","delta","reparsed","d","txStart","Date","now","exec","col","delBody","delPresetFiles","insertPresetFile","del","remove","insert","undefined","values","ctimeMs","parseError","presetName","presets","store","extracted","name","afterReconcile","err","durationMs","prevRaw","prevMax","Number","String","changedSignatureKeys","before","after","keyOf","part","startsWith","split","slice","parse","sig","a","b","changed","val","embedIdentityAdopted","embedPart","find","p","at","indexOf","signatureDiff","label","rebuildContentTable","known"],"mappings":"AAEA,SAASA,eAAe,QAAQ,qBAAqB;AACrD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,uBAAuB;AAEtD,SAASC,QAAQ,QAAQ,iBAAiB;AAE1C,SAASC,SAAS,EAAEC,SAAS,EAAEC,gBAAgB,QAAQ,aAAa;AACpE,SAASC,YAAY,QAAQ,gBAAgB;AAC7C,SAASC,UAAU,EAAEC,OAAO,EAAEC,UAAU,EAAEC,OAAO,QAAQ,cAAc;AAEvE,6FAA6F;AAC7F,4EAA4E;AAC5E,MAAMC,2BAA2B,IAAIC,IAAI;IAAC;IAAQ;IAAU;IAAU;IAAS;CAAe;AAE9F,8FAA8F;AAC9F,MAAMC,0BAA0B;AAEhC,0FAA0F;AAC1F,gDAAgD;AAChD,MAAMC,qBAAqB,CAAC,0KAA0K,CAAC;AAEvM,0FAA0F;AAC1F,2FAA2F;AAC3F,SAASC,iBAAiBC,UAAyB,EAAEC,GAAc,EAAEC,UAAmB;IACtFF,WAAWG,GAAG,CAACF,IAAIG,OAAO,EAAEH,IAAII,MAAM,CAACC,KAAK,EAAEL,IAAII,MAAM,CAACE,OAAO,EAAEN,IAAII,MAAM,CAACG,IAAI,EAAEP,IAAIG,OAAO,EAAEF,aAAaZ,aAAaW,IAAII,MAAM,CAACC,KAAK,IAAI,IAAIJ,aAAaZ,aAAaW,IAAII,MAAM,CAACE,OAAO,IAAI,IAAIL,aAAaZ,aAAaW,IAAII,MAAM,CAACG,IAAI,IAAI;AACrP;AAEA,OAAO,SAASC,UAAUC,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IACtE,MAAMC,QAAQ1B,UAAUwB,KAAKC;IAC7B,MAAME,aAAa,IAAIlB,IAAIiB,MAAME,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IAErD,MAAMa,eAAeP,GAAGQ,OAAO,CAAC,CAAC,iDAAiD,CAAC,EAAEC,GAAG;IAKxF,MAAMC,WAAW,IAAIC,IAAIJ,aAAaF,GAAG,CAAC,CAACO,IAAM;YAACA,EAAEC,IAAI;YAAED;SAAE;IAC5D,MAAME,WAAWP,aAAaQ,MAAM,CAAC,CAACH,IAAM,CAACR,WAAWY,GAAG,CAACJ,EAAEC,IAAI,GAAGR,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IAEtF,MAAMI,YAAYd,MAAMY,MAAM,CAAC,CAACT;QAC9B,MAAMY,MAAMR,SAASS,GAAG,CAACb,EAAEZ,OAAO;QAClC,OAAO,CAACwB,OAAOA,IAAIE,MAAM,KAAKd,EAAEe,OAAO,IAAIH,IAAII,KAAK,KAAKhB,EAAEiB,IAAI;IACjE;IAEA,IAAIT,SAASU,MAAM,KAAK,KAAKP,UAAUO,MAAM,KAAK,GAAG,OAAO;QAAEC,QAAQ;QAAGC,UAAU,EAAE;IAAC;IAEtF,MAAMC,WAAWpD,eAAe0B;IAChC,MAAM2B,cAAc/C,WAAWmB;IAC/B,MAAM6B,aAAuB,EAAE;IAC/B,MAAMC,aAA0B,EAAE;IAClC,MAAMJ,WAAqB,EAAE;IAE7B,oFAAoF;IACpF,uDAAuD;IACvD,MAAMK,SAASvD,SAAS,mBAAmByC,UAAUO,MAAM;IAC3D,IAAIQ,cAAc;IAClB,KAAK,MAAMC,QAAQhB,UAAW;QAC5B,sFAAsF;QACtF,mEAAmE;QACnE,MAAMiB,eAAeP,SAASZ,MAAM,CAAC,CAACoB,UAAY,CAACA,QAAQC,cAAc,IAAID,QAAQC,cAAc,CAACnC,KAAKgC;QACzG,MAAM,EAAE1C,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD,MAAMC;QACxDH,OAAOO,IAAI,CAAC,EAAEN;QACdN,SAASa,IAAI,IAAIF;QACjB,KAAK,MAAMG,OAAOC,OAAOC,IAAI,CAACnD,IAAIoD,IAAI,EAAG;YACvC,IAAI,CAACf,YAAYZ,GAAG,CAACwB,MAAM;gBACzBZ,YAAYgB,GAAG,CAACJ;gBAChBX,WAAWU,IAAI,CAACC;YAClB;QACF;QACAV,WAAWS,IAAI,CAAChD;IAClB;IACAwC,OAAOc,MAAM;IAEb,MAAMC,aAAa;WAAIlB;KAAY;IACnC,uEAAuE;IACvE,sGAAsG;IACtG,IAAIkB,WAAWtB,MAAM,GAAGrC,yBAAyB;QAC/C,MAAM,IAAIb,WACR,gBACA,CAAC,uBAAuB,EAAEwE,WAAWtB,MAAM,CAAC,0EAA0E,EAAErC,wBAAwB,wKAAwK,CAAC;IAE7T;IACA,0FAA0F;IAC1F,sEAAsE;IACtE,MAAM4D,kBAAkBD,WAAW/B,MAAM,CAAC,CAACiC,IAAM/D,yBAAyB+B,GAAG,CAACgC,MAAM,CAACrE,iBAAiBqC,GAAG,CAACgC;IAC1G,sFAAsF;IACtF,gDAAgD;IAChD,MAAMC,YAAY,CAAC,yBAAyB,EAAEF,gBAAgB1C,GAAG,CAACtB,YAAYmE,IAAI,CAAC,MAAM,UAAU,EAAEH,gBAAgB1C,GAAG,CAAC,IAAM,KAAK6C,IAAI,CAAC,MAAM,oCAAoC,EAAEH,gBAClLhC,MAAM,CAAC,CAACiC,IAAMA,MAAM,QACpB3C,GAAG,CAAC,CAAC2C,IAAM,GAAGjE,WAAWiE,GAAG,YAAY,EAAEjE,WAAWiE,IAAI,EACzDE,IAAI,CAAC,OAAO;IAEf,MAAMC,QAAQlC,UAAUF,MAAM,CAAC,CAACT,IAAM,CAACI,SAASM,GAAG,CAACV,EAAEZ,OAAO,GAAGW,GAAG,CAAC,CAACC,IAAMA,EAAEZ,OAAO;IACpF,MAAM0D,QAAwB;QAAEjD;QAAOkD,UAAUvB,WAAWzB,GAAG,CAAC,CAACiD,IAAMA,EAAE5D,OAAO;QAAGyD;QAAOrC;IAAS;IAEnG,MAAMyC,UAAUC,KAAKC,GAAG;IACxBzD,GAAG0D,IAAI,CAAC;IACR,IAAI;YAkD8BvB;QAjDhC,KAAK,MAAMwB,OAAO9B,WAAY7B,GAAG0D,IAAI,CAAC,CAAC,mCAAmC,EAAE3E,WAAW4E,MAAM;QAC7F,6EAA6E;QAC7E,kFAAkF;QAClF,+EAA+E;QAC/E,MAAMC,UAAU5D,GAAGQ,OAAO,CAAC,CAAC,kFAAkF,CAAC;QAC/G,MAAMqD,iBAAiB7D,GAAGQ,OAAO,CAAC,CAAC,yCAAyC,CAAC;QAC7E,MAAMsD,mBAAmB9D,GAAGQ,OAAO,CAAC,CAAC,uDAAuD,CAAC;QAC7F,IAAIM,SAASU,MAAM,GAAG,GAAG;YACvB,MAAMuC,MAAM/D,GAAGQ,OAAO,CAAC,CAAC,wCAAwC,CAAC;YACjE,KAAK,MAAMK,QAAQC,SAAU;oBAMKqB;gBALhC,kFAAkF;gBAClF,qEAAqE;gBACrEyB,QAAQnE,GAAG,CAACoB;gBACZkD,IAAItE,GAAG,CAACoB;gBACRgD,eAAepE,GAAG,CAACoB;gBACnB,KAAK,MAAMsB,WAAWR,UAAUQ,kBAAAA,QAAQ6B,MAAM,cAAd7B,sCAAAA,qBAAAA,SAAiBnC,IAAIa,MAAMuC;YAC7D;QACF;QACA,IAAItB,WAAWN,MAAM,GAAG,GAAG;YACzB,MAAMyC,SAASjE,GAAGQ,OAAO,CAACyC;YAC1B,MAAM3D,aAAaU,GAAGQ,OAAO,CAACpB;YAC9B,wFAAwF;YACxF,+EAA+E;YAC/E,MAAMI,aAAanB,gBAAgB4B,SAASiE;YAC5C,KAAK,MAAM3E,OAAOuC,WAAY;oBAsBIK;gBArBhC,MAAMgC,SAASpB,gBAAgB1C,GAAG,CAAC,CAACsD;wBAO3BpE;oBANP,IAAIoE,QAAQ,QAAQ,OAAOpE,IAAIG,OAAO;oBACtC,IAAIiE,QAAQ,UAAU,OAAOpE,IAAI8B,OAAO;oBACxC,IAAIsC,QAAQ,UAAU,OAAOpE,IAAI6E,OAAO;oBACxC,IAAIT,QAAQ,SAAS,OAAOpE,IAAIgC,IAAI;oBACpC,mFAAmF;oBACnF,IAAIoC,QAAQ,gBAAgB,OAAOpE,IAAI8E,UAAU;oBACjD,QAAO9E,gBAAAA,IAAIoD,IAAI,CAACgB,IAAI,cAAbpE,2BAAAA,gBAAiB;gBAC1B;gBACA,uFAAuF;gBACvF0E,OAAOxE,GAAG,IAAI0E;gBACd,oFAAoF;gBACpF,IAAIzD,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAG;wBAEGyC;oBADhCyB,QAAQnE,GAAG,CAACF,IAAIG,OAAO;oBACvB,KAAK,MAAMyC,WAAWR,UAAUQ,mBAAAA,QAAQ6B,MAAM,cAAd7B,uCAAAA,sBAAAA,SAAiBnC,IAAIT,IAAIG,OAAO,EAAE0D;gBACpE;gBACA/D,iBAAiBC,YAAYC,KAAKC;gBAClC,iFAAiF;gBACjF,2EAA2E;gBAC3E,IAAIkB,SAASM,GAAG,CAACzB,IAAIG,OAAO,GAAGmE,eAAepE,GAAG,CAACF,IAAIG,OAAO;gBAC7D,KAAK,MAAM4E,cAAc/E,IAAIgF,OAAO,CAAET,iBAAiBrE,GAAG,CAACF,IAAIG,OAAO,EAAE4E;gBACxE,KAAK,MAAMnC,WAAWR,UAAUQ,iBAAAA,QAAQqC,KAAK,cAAbrC,qCAAAA,oBAAAA,SAAgBnC,IAAIT,IAAIG,OAAO,EAAEH,IAAIkF,SAAS,CAACtC,QAAQuC,IAAI,CAAC,EAAEtB;YAChG;QACF;QACA,KAAK,MAAMjB,WAAWR,UAAUQ,0BAAAA,QAAQwC,cAAc,cAAtBxC,8CAAAA,6BAAAA,SAAyBnC,IAAIoD;QAC7DpD,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IAEA,qFAAqF;IACrF,8GAA8G;IAC9G,MAAMC,aAAarB,KAAKC,GAAG,KAAKF;IAChC,MAAMuB,UAAUhG,QAAQkB,IAAI;IAC5B,yFAAyF;IACzF,+EAA+E;IAC/E,MAAM+E,UAAUD,YAAY,OAAO,CAAC,IAAIE,OAAOF;IAC/C,IAAID,aAAaE,SAAS/F,QAAQgB,IAAI,oBAAoBiF,OAAOJ;IAEjE,OAAO;QAAEpD,QAAQK,WAAWN,MAAM;QAAEE;IAAS;AAC/C;AAEA,wFAAwF;AACxF,mFAAmF;AACnF,OAAO,SAASwD,qBAAqBC,MAAc,EAAEC,KAAa;IAChE,MAAMC,QAAQ,CAACC,OAAkBA,KAAKC,UAAU,CAAC,aAAaD,KAAKE,KAAK,CAAC,KAAKC,KAAK,CAAC,GAAG,GAAGvC,IAAI,CAAC,OAAOoC,KAAKE,KAAK,CAAC,IAAI,CAAC,EAAE;IACxH,MAAME,QAAQ,CAACC,MAAgB,IAAIhF,IAAIgF,IAAIH,KAAK,CAAC,KAAKnF,GAAG,CAAC,CAACiF,OAAS;gBAACD,MAAMC;gBAAOA;aAAK;IACvF,MAAMM,IAAIF,MAAMP;IAChB,MAAMU,IAAIH,MAAMN;IAChB,MAAMU,UAAU,IAAI5G;IACpB,KAAK,MAAM,CAACsD,KAAKuD,IAAI,IAAIF,EAAG,IAAID,EAAEzE,GAAG,CAACqB,SAASuD,KAAKD,QAAQlD,GAAG,CAACJ;IAChE,KAAK,MAAMA,OAAOoD,EAAElD,IAAI,GAAI,IAAI,CAACmD,EAAE7E,GAAG,CAACwB,MAAMsD,QAAQlD,GAAG,CAACJ;IACzD,OAAOsD;AACT;AAEA,wFAAwF;AACxF,sFAAsF;AACtF,OAAO,SAASE,qBAAqBb,MAAc,EAAEC,KAAa;IAChE,MAAMa,YAAY,CAACN,MAAgBA,IAAIH,KAAK,CAAC,KAAKU,IAAI,CAAC,CAACC,IAAMA,EAAEZ,UAAU,CAAC;IAC3E,MAAMM,IAAII,UAAUd;IACpB,MAAMS,IAAIK,UAAUb;IACpB,IAAIS,MAAM3B,aAAa0B,MAAM1B,WAAW,OAAO;IAC/C,MAAMkC,KAAKR,EAAES,OAAO,CAAC;IACrB,OAAOR,EAAEQ,OAAO,CAAC,SAAS,CAAC,KAAKD,OAAO,CAAC,KAAKR,EAAEH,KAAK,CAAC,GAAGW,QAAQP;AAClE;AAEA,4CAA4C;AAC5C,OAAO,SAASS,cAAcnB,MAAc,EAAEC,KAAa;IACzD,MAAMU,UAAUZ,qBAAqBC,QAAQC;IAC7C,MAAMmB,QAAQ,CAAC/D,MAAiBA,QAAQ,UAAU,mBAAmBA,QAAQ,aAAa,sBAAsBA,IAAI+C,UAAU,CAAC,aAAa,CAAC,QAAQ,EAAE/C,IAAIiD,KAAK,CAAC,GAAG,CAAC,CAAC,GAAG;IACzK,OAAOK,QAAQvE,IAAI,KAAK,IAAI,aAAa;WAAIuE;KAAQ,CAACzF,GAAG,CAACkG,OAAOrD,IAAI,CAAC;AACxE;AAEA,6FAA6F;AAC7F,sFAAsF;AACtF,gGAAgG;AAChG,uFAAuF;AACvF,2FAA2F;AAC3F,kGAAkG;AAClG,OAAO,SAASsD,oBAAoBxG,EAAgB,EAAEC,GAAW,EAAEC,OAAe;IAChF,MAAMuG,QAAQ,IAAIvH,IAAI,AAACc,GAAGQ,OAAO,CAAC,kCAAkCC,GAAG,GAA+BJ,GAAG,CAAC,CAACO,IAAMA,EAAEC,IAAI;IACvH,MAAMV,QAAQ1B,UAAUwB,KAAKC,SAASa,MAAM,CAAC,CAACT,IAAMmG,MAAMzF,GAAG,CAACV,EAAEZ,OAAO;IACvE,MAAMF,aAAanB,gBAAgB4B,SAASiE;IAC5C,MAAM5E,aAAaU,GAAGQ,OAAO,CAACpB;IAC9B,MAAMsC,WAAqB,EAAE;IAC7B1B,GAAG0D,IAAI,CAAC;IACR,IAAI;QACF,KAAK,MAAMzB,QAAQ9B,MAAO;YACxB,MAAM,EAAEZ,GAAG,EAAEmC,UAAUW,YAAY,EAAE,GAAG3D,UAAUuD;YAClDP,SAASa,IAAI,IAAIF;YACjBhD,iBAAiBC,YAAYC,KAAKC;QACpC;QACAQ,GAAG0D,IAAI,CAAC;IACV,EAAE,OAAOkB,KAAK;QACZ5E,GAAG0D,IAAI,CAAC;QACR,MAAMkB;IACR;IACA,OAAOlD;AACT"}

package/dist/esm/embed/cohere.d.ts

@@ -0,0 +1,2 @@
+import type { EmbedProvider } from './types.js';
+export declare function cohereProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider>;

package/dist/esm/embed/cohere.js

@@ -0,0 +1,46 @@
+import { SenseError } from '../errors.js';
+import { fetchWithRetry } from './http.js';
+const BATCH_CAP = 96;
+// Cohere's native /v2/embed: input_type distinguishes doc vs query embeddings, which the
+// OpenAI-compatible shape cannot express.
+export async function cohereProvider(model, url, keyEnv) {
+    const base = (url !== null && url !== void 0 ? url : 'https://api.cohere.com').replace(/\/+$/, '');
+    const headers = {
+        'content-type': 'application/json'
+    };
+    const key = keyEnv ? process.env[keyEnv] : undefined;
+    if (key) headers.authorization = `Bearer ${key}`;
+    async function post(texts, inputType) {
+        const res = await fetchWithRetry(`${base}/v2/embed`, {
+            method: 'POST',
+            headers,
+            body: JSON.stringify({
+                model,
+                input_type: inputType,
+                texts,
+                embedding_types: [
+                    'float'
+                ]
+            })
+        });
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/v2/embed -> HTTP ${res.status}`);
+        const body = await res.json();
+        // Cohere returns one vector per text in order, with no index field to re-sort by; the
+        // count is what a caller mapping positionally still has to be able to trust.
+        const vectors = body.embeddings.float;
+        if (vectors.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/v2/embed returned ${vectors.length} embeddings for ${texts.length} inputs`);
+        return vectors.map((v)=>Float32Array.from(v));
+    }
+    const dims = (await post([
+        'dimension probe'
+    ], 'search_query'))[0].length;
+    return {
+        id: `cohere:${base}:${model}`,
+        dims,
+        batchCap: BATCH_CAP,
+        embedDocuments: (texts)=>post(texts, 'search_document'),
+        embedQuery: async (text)=>(await post([
+                text
+            ], 'search_query'))[0]
+    };
+}

package/dist/esm/embed/cohere.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/cohere.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 96;\n\n// Cohere's native /v2/embed: input_type distinguishes doc vs query embeddings, which the\n// OpenAI-compatible shape cannot express.\nexport async function cohereProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = (url ?? 'https://api.cohere.com').replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], inputType: 'search_document' | 'search_query'): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/v2/embed`, { method: 'POST', headers, body: JSON.stringify({ model, input_type: inputType, texts, embedding_types: ['float'] }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/v2/embed -> HTTP ${res.status}`);\n    const body = (await res.json()) as { embeddings: { float: number[][] } };\n    // Cohere returns one vector per text in order, with no index field to re-sort by; the\n    // count is what a caller mapping positionally still has to be able to trust.\n    const vectors = body.embeddings.float;\n    if (vectors.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/v2/embed returned ${vectors.length} embeddings for ${texts.length} inputs`);\n    return vectors.map((v) => Float32Array.from(v));\n  }\n\n  const dims = (await post(['dimension probe'], 'search_query'))[0].length;\n  return { id: `cohere:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: (texts) => post(texts, 'search_document'), embedQuery: async (text) => (await post([text], 'search_query'))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","cohereProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","inputType","res","method","body","JSON","stringify","input_type","embedding_types","ok","status","json","vectors","embeddings","float","length","map","v","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,yFAAyF;AACzF,0CAA0C;AAC1C,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO,CAACF,gBAAAA,iBAAAA,MAAO,0BAA0BG,OAAO,CAAC,QAAQ;IAC/D,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe,EAAEC,SAA6C;QAChF,MAAMC,MAAM,MAAMjB,eAAe,GAAGM,KAAK,SAAS,CAAC,EAAE;YAAEY,QAAQ;YAAQV;YAASW,MAAMC,KAAKC,SAAS,CAAC;gBAAElB;gBAAOmB,YAAYN;gBAAWD;gBAAOQ,iBAAiB;oBAAC;iBAAQ;YAAC;QAAG;QAC1K,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAIzB,WAAW,eAAe,GAAGO,KAAK,kBAAkB,EAAEW,IAAIQ,MAAM,EAAE;QACzF,MAAMN,OAAQ,MAAMF,IAAIS,IAAI;QAC5B,sFAAsF;QACtF,6EAA6E;QAC7E,MAAMC,UAAUR,KAAKS,UAAU,CAACC,KAAK;QACrC,IAAIF,QAAQG,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAI/B,WAAW,eAAe,GAAGO,KAAK,mBAAmB,EAAEqB,QAAQG,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAC5J,OAAOH,QAAQI,GAAG,CAAC,CAACC,IAAMC,aAAaC,IAAI,CAACF;IAC9C;IAEA,MAAMG,OAAO,AAAC,CAAA,MAAMrB,KAAK;QAAC;KAAkB,EAAE,eAAc,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxE,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAE9B,KAAK,CAAC,EAAEH,OAAO;QAAEgC;QAAME,UAAUpC;QAAWqC,gBAAgB,CAACvB,QAAUD,KAAKC,OAAO;QAAoBwB,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM1B,KAAK;gBAAC0B;aAAK,EAAE,eAAc,CAAE,CAAC,EAAE;IAAC;AACpM"}

package/dist/esm/embed/http.d.ts

@@ -0,0 +1,2 @@
+export declare function fetchWithRetry(url: string, init: RequestInit, baseDelayMs?: number): Promise<Response>;
+export declare function probeReachable(url: string, timeoutMs?: number): Promise<boolean>;

package/dist/esm/embed/http.js

@@ -0,0 +1,33 @@
+const MAX_ATTEMPTS = 3;
+function sleep(ms) {
+    return new Promise((resolve)=>setTimeout(resolve, ms));
+}
+// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential
+// backoff honoring Retry-After; any other status returns immediately for the caller to handle.
+export async function fetchWithRetry(url, init, baseDelayMs = 200) {
+    for(let attempt = 0; attempt < MAX_ATTEMPTS; attempt++){
+        try {
+            const res = await fetch(url, init);
+            if (res.ok || res.status !== 429 && res.status < 500 || attempt === MAX_ATTEMPTS - 1) return res;
+            const retryAfter = Number(res.headers.get('retry-after'));
+            // Capped: a server naming a huge Retry-After must not stall an embed run for it.
+            await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10000) : baseDelayMs * 2 ** attempt);
+        } catch (err) {
+            if (attempt === MAX_ATTEMPTS - 1) throw err;
+            await sleep(baseDelayMs * 2 ** attempt);
+        }
+    }
+    throw new Error('unreachable');
+}
+// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as
+// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.
+export async function probeReachable(url, timeoutMs = 1500) {
+    try {
+        await fetch(url, {
+            signal: AbortSignal.timeout(timeoutMs)
+        });
+        return true;
+    } catch  {
+        return false;
+    }
+}

package/dist/esm/embed/http.js.map

@@ -0,0 +1 @@
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    try {\n      const res = await fetch(url, init);\n      if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n      const retryAfter = Number(res.headers.get('retry-after'));\n      // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n      await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n    } catch (err) {\n      if (attempt === MAX_ATTEMPTS - 1) throw err;\n      await sleep(baseDelayMs * 2 ** attempt);\n    }\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n"],"names":["MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","fetchWithRetry","url","init","baseDelayMs","attempt","res","fetch","ok","status","retryAfter","Number","headers","get","Math","min","err","Error","probeReachable","timeoutMs","signal","AbortSignal","timeout"],"mappings":"AAAA,MAAMA,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,2FAA2F;AAC3F,+FAA+F;AAC/F,OAAO,eAAeI,eAAeC,GAAW,EAAEC,IAAiB,EAAEC,cAAc,GAAG;IACpF,IAAK,IAAIC,UAAU,GAAGA,UAAUV,cAAcU,UAAW;QACvD,IAAI;YACF,MAAMC,MAAM,MAAMC,MAAML,KAAKC;YAC7B,IAAIG,IAAIE,EAAE,IAAKF,IAAIG,MAAM,KAAK,OAAOH,IAAIG,MAAM,GAAG,OAAQJ,YAAYV,eAAe,GAAG,OAAOW;YAC/F,MAAMI,aAAaC,OAAOL,IAAIM,OAAO,CAACC,GAAG,CAAC;YAC1C,iFAAiF;YACjF,MAAMjB,MAAMc,aAAa,IAAII,KAAKC,GAAG,CAACL,aAAa,MAAM,SAAUN,cAAc,KAAKC;QACxF,EAAE,OAAOW,KAAK;YACZ,IAAIX,YAAYV,eAAe,GAAG,MAAMqB;YACxC,MAAMpB,MAAMQ,cAAc,KAAKC;QACjC;IACF;IACA,MAAM,IAAIY,MAAM;AAClB;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,OAAO,eAAeC,eAAehB,GAAW,EAAEiB,YAAY,IAAI;IAChE,IAAI;QACF,MAAMZ,MAAML,KAAK;YAAEkB,QAAQC,YAAYC,OAAO,CAACH;QAAW;QAC1D,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF"}

package/dist/esm/embed/identity.d.ts

@@ -0,0 +1,13 @@
+export declare const MODEL_FILES: string[];
+export declare const MODEL_FILENAMES: string;
+export declare function isDownloadable(model: string): boolean;
+export declare function refsMainPath(model: string): string;
+export declare function readRef(model: string): string | undefined;
+export declare function writeRef(model: string, sha: string): void;
+export declare function snapshotDir(model: string, sha: string): string;
+export declare function languagesPath(model: string): string;
+export declare function readLanguages(model: string): string[] | undefined;
+export declare function writeLanguages(model: string, languages: string[]): void;
+export declare function modelDir(model: string): string;
+export declare function hasModelFiles(model: string): boolean;
+export declare function modelIdentity(model: string): string | undefined;

package/dist/esm/embed/identity.js

@@ -0,0 +1,92 @@
+import { existsSync, mkdirSync, readFileSync, statSync, writeFileSync } from 'node:fs';
+import { homedir } from 'node:os';
+import { join } from 'node:path';
+// HF cache layout, no network: ~/.sense/models/models--<org>--<name>/{refs/main,
+// snapshots/<sha>/<files>}, files stored directly (hf_hub's blobs/symlinks layer skipped).
+export const MODEL_FILES = [
+    'model.safetensors',
+    'tokenizer.json'
+];
+// The pair, for messages that name what a local model directory must contain.
+export const MODEL_FILENAMES = MODEL_FILES.join(' and ');
+// A Hugging Face repo id: one slash, HF's charset. Anything else is a path, used as one rather
+// than resolved against the HF cache.
+const HF_MODEL_ID = /^[A-Za-z0-9][\w.-]*\/[A-Za-z0-9][\w.-]*$/;
+export function isDownloadable(model) {
+    return HF_MODEL_ID.test(model);
+}
+function repoDir(model) {
+    const [org, name] = model.split('/');
+    return join(homedir(), '.sense', 'models', `models--${org}--${name}`);
+}
+export function refsMainPath(model) {
+    return join(repoDir(model), 'refs', 'main');
+}
+// What `main` last resolved to, read from disk -- no network. Undefined until some resolution
+// (a download or an explicit prefetch) has written it once.
+export function readRef(model) {
+    const p = refsMainPath(model);
+    if (!existsSync(p)) return undefined;
+    const sha = readFileSync(p, 'utf8').trim();
+    return sha.length > 0 ? sha : undefined;
+}
+// A recorded ref pins until its snapshot directory is removed; nothing here re-resolves it.
+export function writeRef(model, sha) {
+    mkdirSync(join(repoDir(model), 'refs'), {
+        recursive: true
+    });
+    writeFileSync(refsMainPath(model), sha);
+}
+export function snapshotDir(model, sha) {
+    return join(repoDir(model), 'snapshots', sha);
+}
+// Languages describe the repo, not one weight snapshot, so this lives beside refs/, not
+// inside a snapshot dir.
+export function languagesPath(model) {
+    return join(repoDir(model), 'languages.json');
+}
+// undefined means "never resolved yet"; an empty array is a resolved fact (checked, none
+// declared) and is cached the same way, so a languageless model is not re-fetched every run.
+export function readLanguages(model) {
+    const p = languagesPath(model);
+    if (!existsSync(p)) return undefined;
+    try {
+        return JSON.parse(readFileSync(p, 'utf8'));
+    } catch  {
+        return undefined;
+    }
+}
+export function writeLanguages(model, languages) {
+    mkdirSync(repoDir(model), {
+        recursive: true
+    });
+    writeFileSync(languagesPath(model), JSON.stringify(languages));
+}
+// A local directory the caller manages, or the resolved snapshot once `main` is pinned; an HF
+// id with no recorded ref yet has no known snapshot, so this names the (fileless) repo dir.
+export function modelDir(model) {
+    if (!isDownloadable(model)) return model;
+    const sha = readRef(model);
+    return sha ? snapshotDir(model, sha) : repoDir(model);
+}
+// Both files, so an interrupted download reads as absent and the next one resumes it.
+export function hasModelFiles(model) {
+    const dir = modelDir(model);
+    return MODEL_FILES.every((file)=>existsSync(join(dir, file)));
+}
+// Weight identity for the signature, no network: a HF id's resolved sha, or the two
+// files' size+mtime for a local path. Undefined when nothing is known yet.
+export function modelIdentity(model) {
+    if (isDownloadable(model)) return readRef(model);
+    const dir = modelDir(model);
+    const parts = [];
+    for (const file of MODEL_FILES){
+        try {
+            const st = statSync(join(dir, file));
+            parts.push(`${st.size}:${Math.trunc(st.mtimeMs)}`);
+        } catch  {
+            return undefined;
+        }
+    }
+    return parts.join(',');
+}