sensemaking 0.18.3 → 0.19.0

package/dist/cjs/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise\n// only need store.ts's network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string): Promise<string> {\n  const existing = readRef(model);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has\n// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an\n// array here). Tags are a fallback for cards that only tag languages, filtered to recognized\n// codes so an unrelated short tag (format markers etc.) is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Best-effort, cached forever once resolved (even to \"none\"): unlike model weights, missing or\n// unreachable metadata is not an error -- it just means the language-fit check stays off.\nasync function ensureLanguages(model: string): Promise<void> {\n  if (existsSync(languagesPath(model))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    return;\n  }\n  if (!res.ok) return;\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? []);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  const dir = snapshotDir(model, sha);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model);\n  return downloadModelRevision(model, sha, onFile);\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","downloadModel","downloadModelRevision","hasModelFiles","isDownloadable","localModelMissing","modelDir","modelPresent","readLanguages","refsMainPath","snapshotDir","writeLanguages","writeRef","cfg","e","embedConfig","provider","model","resolveSha","existing","url","res","err","sha","readRef","fetchWithRetry","method","redirect","SenseError","message","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","isKnownLanguageTag","undefined","ensureLanguages","measured","existsSync","languagesPath","MEASURED_MODEL_LANGUAGES","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","writeFileAtomic","onFile","dir","file","mkdirSync","recursive","join"],"mappings":";;;;;;;;;;;QAawCA;eAAAA,2BAAe;;QAAEC;eAAAA,uBAAW;;QA2G9CC;eAAAA;;QAbAC;eAAAA;;QA9FbC;eAAAA,yBAAa;;QAAEC;eAAAA,0BAAc;;QActBC;eAAAA;;QAdsDC;eAAAA,oBAAQ;;QAI9DC;eAAAA;;QAJgEC;eAAAA,yBAAa;;QAAEC;eAAAA,wBAAY;;QAAEC;eAAAA,uBAAW;;QAAEC;eAAAA,0BAAc;;QAAEC;eAAAA,oBAAQ;;;sBAb5G;wBACjB;uBAEO;wBACD;6BACK;sBACD;0BAC2F;2BAC7D;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAStD,SAASL,aAAaM,GAAW;IACtC,IAAMC,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,QAAQ,KAAK,YAAYb,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACzD;AAMO,SAASZ,kBAAkBS,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEE,QAAQ,KAAK,YAAY,CAACZ,IAAAA,0BAAc,EAACU,EAAEG,KAAK,KAAK,CAACd,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,SAAeC,WAAWD,KAAa;;YAC/BE,UAEAC,KAGFC,KAGKC,KAGHC;;;;oBAXAJ,WAAWK,IAAAA,mBAAO,EAACP;oBACzB,IAAIE,UAAU;;wBAAOA;;oBACfC,MAAM,AAAC,0BAA+B,OAANH,OAAM;;;;;;;;;oBAKpC;;wBAAMQ,IAAAA,sBAAc,EAACL,KAAK;4BAAEM,QAAQ;4BAAQC,UAAU;wBAAS;;;oBAArEN,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAARR,KAAI,MAA2B,OAAvB,AAACE,IAAcO,OAAO;;oBAEjFN,MAAMF,IAAIS,OAAO,CAACC,GAAG,CAAC;oBAC5B,IAAI,CAACR,KAAK,MAAM,IAAIK,oBAAU,CAAC,eAAe,AAAC,GAAM,OAAJR,KAAI;oBACrDR,IAAAA,oBAAQ,EAACK,OAAOM;oBAChB;;wBAAOA;;;;IACT;;AAOA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,mFAAmF;AACnF,SAASS,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,IAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,IAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACC,+BAAkB;IAC5D,OAAOH,SAASD,MAAM,GAAG,IAAIC,WAAWI;AAC1C;AAEA,+FAA+F;AAC/F,0FAA0F;AAC1F,SAAeC,gBAAgB5B,KAAa;;YAqBpBe,oBAlBhBc,UAKFzB,aAOAY;;;;oBAdJ,IAAIc,IAAAA,kBAAU,EAACC,IAAAA,yBAAa,EAAC/B,SAAS;;;oBACtC,kFAAkF;oBAC5E6B,WAAWG,qCAAwB,CAAChC,MAAM;oBAChD,IAAI6B,UAAU;wBACZnC,IAAAA,0BAAc,EAACM,OAAO6B;wBACtB;;;oBACF;;;;;;;;;oBAGQ;;wBAAMrB,IAAAA,sBAAc,EAAC,AAAC,qCAA0C,OAANR,QAAS,CAAC;;;oBAA1EI,MAAM;;;;;;;oBAEN;;;;oBAEF,IAAI,CAACA,IAAI6B,EAAE,EAAE;;;;;;;;;;;oBAGH;;wBAAM7B,IAAI8B,IAAI;;;oBAAtBlB,OAAQ;;;;;;;oBAER;;;;oBAEFtB,IAAAA,0BAAc,EAACM,QAAOe,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA;;;;;;IACxB;;AAEA,SAAeoB,YAAYhC,GAAW,EAAEiC,IAAY;;YAC9ChC,KAGKC;;;;;;;;;;oBADD;;wBAAMG,IAAAA,sBAAc,EAACL,KAAK,CAAC;;;oBAAjCC,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAARR,KAAI,MAA2B,OAAvB,AAACE,IAAcO,OAAO;;oBAEvF,IAAI,CAACR,IAAI6B,EAAE,EAAE,MAAM,IAAItB,oBAAU,CAAC,eAAe,AAAC,GAAiBP,OAAfD,KAAI,aAAsB,OAAXC,IAAIiC,MAAM;;wBAC7DD;;wBAAME,OAAOC,IAAI;oBAAC;;wBAAMnC,IAAIoC,WAAW;;;oBAAvDC,8BAAe;wBAAOH,QAAAA;4BAAY;;;;;;;;IACpC;;AAIO,SAAerD,sBAAsBe,KAAa,EAAEM,GAAW,EAAEoC,MAA4C;;YAC5GC,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;oBAFLD,MAAMlD,IAAAA,uBAAW,EAACO,OAAOM;oBAC/BuC,IAAAA,iBAAS,EAACF,KAAK;wBAAEG,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAc/D,uBAAW;;;2BAAzB,6BAAA,QAAA;;;;oBAAM6D,OAAN;oBACH,IAAId,IAAAA,kBAAU,EAACiB,IAAAA,cAAI,EAACJ,KAAKC,QAAQ;;;;oBACjCF,mBAAAA,6BAAAA,OAASE,MAAMD;oBACf;;wBAAMR,YAAY,AAAC,0BAA0C7B,OAAjBN,OAAM,aAAkB4C,OAAPtC,KAAI,KAAQ,OAALsC,OAAQG,IAAAA,cAAI,EAACJ,KAAKC;;;oBAAtF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOD;;;;IACT;;AAIO,SAAe3D,cAAcgB,KAAa,EAAE0C,MAA4C;;YAIvFpC;;;;oBAHN,IAAI,CAACnB,IAAAA,0BAAc,EAACa,QAAQ;wBAC1B,MAAM,IAAIW,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANX,OAAM;oBAC5D;oBACY;;wBAAMC,WAAWD;;;oBAAvBM,MAAM;oBACZ,sFAAsF;oBACtF,oFAAoF;oBACpF;;wBAAMsB,gBAAgB5B;;;oBAAtB;oBACA;;wBAAOf,sBAAsBe,OAAOM,KAAKoC;;;;IAC3C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","downloadModel","downloadModelRevision","hasModelFiles","isDownloadable","localModelMissing","modelDir","modelPresent","readLanguages","refsMainPath","snapshotDir","writeLanguages","writeRef","cfg","e","embedConfig","provider","model","resolveSha","root","existing","url","res","err","sha","DEFAULT_MODELS_ROOT","readRef","fetchWithRetry","method","redirect","SenseError","message","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","isKnownLanguageTag","undefined","cardUnreadable","console","error","ensureLanguages","measured","existsSync","languagesPath","MEASURED_MODEL_LANGUAGES","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","writeFileAtomic","onFile","dir","file","mkdirSync","recursive","join"],"mappings":";;;;;;;;;;;QAYwCA;eAAAA,2BAAe;;QAAEC;eAAAA,uBAAW;;QAkH9CC;eAAAA;;QAbAC;eAAAA;;QArGbC;eAAAA,yBAAa;;QAAEC;eAAAA,0BAAc;;QActBC;eAAAA;;QAdsDC;eAAAA,oBAAQ;;QAI9DC;eAAAA;;QAJgEC;eAAAA,yBAAa;;QAAEC;eAAAA,wBAAY;;QAAEC;eAAAA,uBAAW;;QAAEC;eAAAA,0BAAc;;QAAEC;eAAAA,oBAAQ;;;sBAZ5G;wBACjB;uBAEO;wBACD;6BACK;sBACD;0BACgH;2BAClF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQtD,SAASL,aAAaM,GAAW;IACtC,IAAMC,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,QAAQ,KAAK,YAAYb,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACzD;AAMO,SAASZ,kBAAkBS,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEE,QAAQ,KAAK,YAAY,CAACZ,IAAAA,0BAAc,EAACU,EAAEG,KAAK,KAAK,CAACd,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,SAAeC;wCAAWD,KAAa;YAAEE,MACjCC,UAEAC,KAGFC,KAGKC,KAGHC;;;;;oBAZiCL,OAAAA,oEAAeM,+BAAmB;oBACnEL,WAAWM,IAAAA,mBAAO,EAACT,OAAOE;oBAChC,IAAIC,UAAU;;wBAAOA;;oBACfC,MAAM,AAAC,0BAA+B,OAANJ,OAAM;;;;;;;;;oBAKpC;;wBAAMU,IAAAA,sBAAc,EAACN,KAAK;4BAAEO,QAAQ;4BAAQC,UAAU;wBAAS;;;oBAArEP,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIO,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAART,KAAI,MAA2B,OAAvB,AAACE,IAAcQ,OAAO;;oBAEjFP,MAAMF,IAAIU,OAAO,CAACC,GAAG,CAAC;oBAC5B,IAAI,CAACT,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,GAAM,OAAJT,KAAI;oBACrDT,IAAAA,oBAAQ,EAACK,OAAOO,KAAKL;oBACrB;;wBAAOK;;;;IACT;;AAOA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASU,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,IAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,IAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACC,+BAAkB;IAC5D,OAAOH,SAASD,MAAM,GAAG,IAAIC,WAAWI;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASC,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,AAAC,4CAAiD,OAANhC,OAAM;AAClE;AAEA,SAAeiC;wCAAgBjC,KAAa;YAAEE,MA0BtBe,oBAvBhBiB,UAKF7B,aAWAa;;;;;oBAnBwChB,OAAAA,oEAAeM,+BAAmB;oBAC9E,IAAI2B,IAAAA,kBAAU,EAACC,IAAAA,yBAAa,EAACpC,OAAOE,QAAQ;;;oBAC5C,kFAAkF;oBAC5EgC,WAAWG,qCAAwB,CAACrC,MAAM;oBAChD,IAAIkC,UAAU;wBACZxC,IAAAA,0BAAc,EAACM,OAAOkC,UAAUhC;wBAChC;;;oBACF;;;;;;;;;oBAGQ;;wBAAMQ,IAAAA,sBAAc,EAAC,AAAC,qCAA0C,OAANV,QAAS,CAAC;;;oBAA1EK,MAAM;;;;;;;oBAENyB,eAAe9B;oBACf;;;;oBAEF,IAAI,CAACK,IAAIiC,EAAE,EAAE;wBACXR,eAAe9B;wBACf;;;oBACF;;;;;;;;;oBAGU;;wBAAMK,IAAIkC,IAAI;;;oBAAtBrB,OAAQ;;;;;;;oBAERY,eAAe9B;oBACf;;;;oBAEFN,IAAAA,0BAAc,EAACM,QAAOiB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,yBAA+Bf;;;;;;IACvD;;AAEA,SAAesC,YAAYpC,GAAW,EAAEqC,IAAY;;YAC9CpC,KAGKC;;;;;;;;;;oBADD;;wBAAMI,IAAAA,sBAAc,EAACN,KAAK,CAAC;;;oBAAjCC,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIO,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAART,KAAI,MAA2B,OAAvB,AAACE,IAAcQ,OAAO;;oBAEvF,IAAI,CAACT,IAAIiC,EAAE,EAAE,MAAM,IAAIzB,oBAAU,CAAC,eAAe,AAAC,GAAiBR,OAAfD,KAAI,aAAsB,OAAXC,IAAIqC,MAAM;;wBAC7DD;;wBAAME,OAAOC,IAAI;oBAAC;;wBAAMvC,IAAIwC,WAAW;;;oBAAvDC,8BAAe;wBAAOH,QAAAA;4BAAY;;;;;;;;IACpC;;AAIO,SAAe1D;wCAAsBe,KAAa,EAAEO,GAAW,EAAEwC,MAA4C;YAAE7C,MAC9G8C,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;;oBAHyG/C,OAAAA,oEAAeM,+BAAmB;oBAChJwC,MAAMvD,IAAAA,uBAAW,EAACO,OAAOO,KAAKL;oBACpCgD,IAAAA,iBAAS,EAACF,KAAK;wBAAEG,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcpE,uBAAW;;;2BAAzB,6BAAA,QAAA;;;;oBAAMkE,OAAN;oBACH,IAAId,IAAAA,kBAAU,EAACiB,IAAAA,cAAI,EAACJ,KAAKC,QAAQ;;;;oBACjCF,mBAAAA,6BAAAA,OAASE,MAAMD;oBACf;;wBAAMR,YAAY,AAAC,0BAA0CjC,OAAjBP,OAAM,aAAkBiD,OAAP1C,KAAI,KAAQ,OAAL0C,OAAQG,IAAAA,cAAI,EAACJ,KAAKC;;;oBAAtF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOD;;;;IACT;;AAIO,SAAehE;wCAAcgB,KAAa,EAAE+C,MAA4C;YAAE7C,MAIzFK;;;;;oBAJyFL,OAAAA,oEAAeM,+BAAmB;oBACjI,IAAI,CAACrB,IAAAA,0BAAc,EAACa,QAAQ;wBAC1B,MAAM,IAAIa,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANb,OAAM;oBAC5D;oBACY;;wBAAMC,WAAWD,OAAOE;;;oBAA9BK,MAAM;oBACZ,sFAAsF;oBACtF,oFAAoF;oBACpF;;wBAAM0B,gBAAgBjC,OAAOE;;;oBAA7B;oBACA;;wBAAOjB,sBAAsBe,OAAOO,KAAKwC,QAAQ7C;;;;IACnD"}

package/dist/cjs/embed/types.js

@@ -1,7 +1,5 @@
 // Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
-// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]
-// column width, so both stores share one source of truth for it rather than a schema
-// constant drifting from the quantization one.
+// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true

package/dist/cjs/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]\n// column width, so both stores share one source of truth for it rather than a schema\n// constant drifting from the quantization one.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,2FAA2F;AAC3F,qFAAqF;AACrF,+CAA+C;;;;;+BAClCA;;;eAAAA;;;AAAN,IAAMA,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;;;;;+BACrFA;;;eAAAA;;;AAAN,IAAMA,aAAa"}

package/dist/cjs/errors.d.cts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING' | 'LEXICAL_NULL_PATH';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING' | 'LEXICAL_NULL_PATH';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/cjs/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAsBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING'\n  | 'LEXICAL_NULL_PATH';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","code","message","name","Error"],"mappings":"AAAA,qDAAqD;;;;;+BAuBxCA;;;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAAN,IAAA,AAAMA,2BAAN;;cAAMA;aAAAA,WAGCC,IAAoB,EAAEC,OAAe;gCAHtCF;;gBAIT,kBAJSA;YAIHE;;QACN,MAAKC,IAAI,GAAG;QACZ,MAAKF,IAAI,GAAGA;;;WANHD;qBAAmBI"}

package/dist/cjs/features/embed.js

@@ -10,7 +10,9 @@ Object.defineProperty(exports, "embed", {
 });
 var _indexts = require("../chunk/index.js");
 var _indexts1 = require("../config/index.js");
+var _handoffts = require("../embed/handoff.js");
 var _identityts = require("../embed/identity.js");
+var _frontmatterts = require("../scan/frontmatter.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -136,23 +138,16 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
-// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary
-// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.
-//
-// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw
-// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw
-// put the frontmatter block in its own leading chunk on every note with a heading, which made
-// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also
-// disagreed with FTS, which indexes the body alone.
+// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
+// `offset` shifts line numbers back onto raw and a range stays a direct Read range.
 function chunksOf(blocks, body, search) {
     var offset = arguments.length > 3 && arguments[3] !== void 0 ? arguments[3] : 0, chunkTokens = arguments.length > 4 ? arguments[4] : void 0;
     var prefix = [
         search === null || search === void 0 ? void 0 : search.title,
         search === null || search === void 0 ? void 0 : search.summary
     ].filter(Boolean).join('\n');
-    // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change
-    // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker
-    // input not already carried in config.
+    // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature
+    // for every chunker input not already carried in config.
     return (0, _indexts.chunkFromBlocks)(blocks, body, chunkTokens !== undefined ? {
         targetTokens: chunkTokens
     } : undefined).map(function(c) {
@@ -187,7 +182,7 @@ var embed = {
         var _cfg_embed;
         // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.
         // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
-        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : (0, _indexts.parse)(body), body, search, raw.split('\n').length - body.split('\n').length, cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
+        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : (0, _indexts.parse)(body), body, search, (0, _frontmatterts.countLines)(raw) - (0, _frontmatterts.countLines)(body), cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
     remove: function remove(db, paths) {
         return _async_to_generator(function() {
@@ -218,17 +213,19 @@ var embed = {
     // filter skips it), so extracted is undefined here -- those docs contribute no rows.
     function store(db, docs) {
         return _async_to_generator(function() {
-            var rows, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step;
+            var rows, texts, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _loop, _iterator, _step;
             return _ts_generator(this, function(_state) {
                 switch(_state.label){
                     case 0:
                         rows = [];
+                        texts = new Map();
                         _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                         try {
                             _loop = function() {
                                 var _step_value = _step.value, path = _step_value.path, extracted = _step_value.extracted;
                                 if (!extracted) return "continue";
-                                extracted.forEach(function(c, idx) {
+                                var chunks = extracted;
+                                chunks.forEach(function(c, idx) {
                                     return rows.push([
                                         path,
                                         idx,
@@ -236,6 +233,9 @@ var embed = {
                                         c.endLine
                                     ]);
                                 });
+                                texts.set(path, chunks.map(function(c) {
+                                    return c.text;
+                                }));
                             };
                             for(_iterator = docs[Symbol.iterator](); !(_iteratorNormalCompletion = (_step = _iterator.next()).done); _iteratorNormalCompletion = true)_loop();
                         } catch (err) {
@@ -261,6 +261,9 @@ var embed = {
                         ];
                     case 1:
                         _state.sent();
+                        // The text extract() just computed, handed to the embed pass that follows in this same
+                        // process rather than re-derived from the file there (src/embed/handoff.ts).
+                        (0, _handoffts.stashChunkText)(db, texts);
                         return [
                             2
                         ];
@@ -271,9 +274,8 @@ var embed = {
     enabledForFile: function enabledForFile(_cfg, file) {
         return file.embed;
     },
-    // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it "embed
-    // settings". Driven by embedConfig(cfg) directly (a model can be configured with no preset
-    // yet using vectors), not by whether this feature is itself active.
+    // Keyed so a rebuild notice can name it "embed settings". Driven by embedConfig(cfg) directly:
+    // a model can be configured with no preset yet using vectors.
     signature: function signature(cfg) {
         var e = (0, _indexts1.embedConfig)(cfg);
         if (!e) return 'embed:off';

package/dist/cjs/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded. Reconcile writes dirty\n// rows; embedding tops up on the next search, so staleness costs recall. Providers, the vector\n// fill, and the search-time helpers live under src/embed/. Block/group structure (D1-D5) lives\n// in src/chunk; this keeps only the title/summary prefix and the empty-chunk drop (D6).\n\nexport type { Chunk };\n\n// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary\n// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change\n  // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker\n  // input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, raw.split('\\n').length - body.split('\\n').length, cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      (extracted as Chunk[]).forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n    }\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it \"embed\n  // settings\". Driven by embedConfig(cfg) directly (a model can be configured with no preset\n  // yet using vectors), not by whether this feature is itself active.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["embed","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","chunkFromBlocks","undefined","targetTokens","map","c","startLine","endLine","text","name","schema","db","exec","extract","raw","_data","cfg","parse","split","length","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push","enabledForFile","_cfg","file","signature","e","embedConfig","identity","provider","modelIdentity","model","chunkVersion","CHUNK_VERSION"],"mappings":";;;;+BAiCaA;;;eAAAA;;;uBAhCyC;wBAC1B;0BACE;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAU9B,6FAA6F;AAC7F,iGAAiG;AACjG,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C;QAAEC,SAAAA,iEAAS,GAAGC;IACxG,IAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,2FAA2F;IAC3F,yFAAyF;IACzF,uCAAuC;IACvC,OAAOC,IAAAA,wBAAe,EAACX,QAAQC,MAAMG,gBAAgBQ,YAAY;QAAEC,cAAcT;IAAY,IAAIQ,WAAWE,GAAG,CAAC,SAACC;eAAO;YACtHC,WAAWD,EAAEC,SAAS,GAAGb;YACzBc,SAASF,EAAEE,OAAO,GAAGd;YACrBe,MAAMb,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAW,OAAPU,EAAEG,IAAI,IAAKH,EAAEG,IAAI;QAChD;;AACF;AAEO,IAAMpB,QAAiB;IAC5BqB,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG4D0B;QAFvG,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAU2B,IAAAA,cAAK,EAAC1B,OAAOA,MAAMC,QAAQsB,IAAII,KAAK,CAAC,MAAMC,MAAM,GAAG5B,KAAK2B,KAAK,CAAC,MAAMC,MAAM,EAAEH,gBAAAA,2BAAAA,aAAAA,IAAK5B,KAAK,cAAV4B,iCAAAA,WAAYtB,WAAW;IAChI;IACM0B,QAAN,SAAMA,OAAOT,EAAE,EAAEU,KAAK;;;;;wBACpB,IAAIA,MAAMF,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMR,GAAGW,QAAQ,CACf,2CACAD,MAAMjB,GAAG,CAAC,SAACmB;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IAGMC,OAFN,8FAA8F;IAC9F,qFAAqF;IACrF,SAAMA,MAAMb,EAAE,EAAEc,IAAI;;gBACZC,MACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBADCA;wBACD,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCACjB,IAAI,CAACA,WAAW,OAAA;gCACfA,UAAsBC,OAAO,CAAC,SAACxB,GAAGyB;2CAAQJ,KAAKK,IAAI,CAAC;wCAACJ;wCAAMG;wCAAKzB,EAAEC,SAAS;wCAAED,EAAEE,OAAO;qCAAC;;4BAC1F;4BAHA,IAAK,YAA6BkB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBAIL,IAAIC,KAAKP,MAAM,KAAK,GAAG;;;wBACvB;;4BAAMR,GAAGW,QAAQ,CAAC,+GAA+GI;;;wBAAjI;;;;;;QACF;;IACAM,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK9C,KAAK;IACnB;IACA,sFAAsF;IACtF,2FAA2F;IAC3F,oEAAoE;IACpE+C,WAAAA,SAAAA,UAAUnB,GAAG;QACX,IAAMoB,IAAIC,IAAAA,qBAAW,EAACrB;QACtB,IAAI,CAACoB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,IAAME,WAAWF,EAAEG,QAAQ,KAAK,WAAWC,IAAAA,yBAAa,EAACJ,EAAEK,KAAK,IAAIvC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,IAAMwC,eAAeN,EAAE1C,WAAW,KAAKQ,YAAY,AAAC,GAAmBkC,OAAjBO,sBAAa,EAAC,KAAiB,OAAdP,EAAE1C,WAAW,IAAKiD,sBAAa;QACtG,OAAO,AAAC,SAAsBP,OAAdA,EAAEG,QAAQ,EAAC,KAAcG,OAAXN,EAAEK,KAAK,EAAC,KAAkBH,OAAfI,cAA4D,OAA7CJ,aAAapC,YAAY,AAAC,IAAY,OAAToC,YAAa;IACpG;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and embedding tops up on the next search, so staleness costs recall.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, string[]>();\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      const chunks = extracted as Chunk[];\n      chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n      texts.set(\n        path,\n        chunks.map((c) => c.text)\n      );\n    }\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n    // The text extract() just computed, handed to the embed pass that follows in this same\n    // process rather than re-derived from the file there (src/embed/handoff.ts).\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["embed","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","chunkFromBlocks","undefined","targetTokens","map","c","startLine","endLine","text","name","schema","db","exec","extract","raw","_data","cfg","parse","countLines","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","stashChunkText","enabledForFile","_cfg","file","signature","e","embedConfig","identity","provider","modelIdentity","model","chunkVersion","CHUNK_VERSION"],"mappings":";;;;+BA0BaA;;;eAAAA;;;uBAzByC;wBAC1B;yBACG;0BACD;6BACH;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQ3B,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C;QAAEC,SAAAA,iEAAS,GAAGC;IACxG,IAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOC,IAAAA,wBAAe,EAACX,QAAQC,MAAMG,gBAAgBQ,YAAY;QAAEC,cAAcT;IAAY,IAAIQ,WAAWE,GAAG,CAAC,SAACC;eAAO;YACtHC,WAAWD,EAAEC,SAAS,GAAGb;YACzBc,SAASF,EAAEE,OAAO,GAAGd;YACrBe,MAAMb,SAAS,AAAC,GAAaU,OAAXV,QAAO,MAAW,OAAPU,EAAEG,IAAI,IAAKH,EAAEG,IAAI;QAChD;;AACF;AAEO,IAAMpB,QAAiB;IAC5BqB,MAAM;IACAC,QAAN,SAAMA,OAAOC,EAAE;;;;;wBACb;;4BAAMA,GAAGC,IAAI,CAAC;;;wBAAd;;;;;;QACF;;IACAC,SAAAA,SAAAA,QAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAU2B,IAAAA,cAAK,EAAC1B,OAAOA,MAAMC,QAAQ0B,IAAAA,yBAAU,EAACJ,OAAOI,IAAAA,yBAAU,EAAC3B,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAK5B,KAAK,cAAV4B,iCAAAA,WAAYtB,WAAW;IAClH;IACMyB,QAAN,SAAMA,OAAOR,EAAE,EAAES,KAAK;;;;;wBACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;;;wBACxB;;4BAAMV,GAAGW,QAAQ,CACf,2CACAF,MAAMhB,GAAG,CAAC,SAACmB;uCAAM;oCAACA;iCAAE;;;;wBAFtB;;;;;;QAIF;;IAGMC,OAFN,8FAA8F;IAC9F,qFAAqF;IACrF,SAAMA,MAAMb,EAAE,EAAEc,IAAI;;gBACZC,MACAC,OACD,2BAAA,mBAAA,uBAAA,WAAA;;;;wBAFCD;wBACAC,QAAQ,IAAIC;wBACb,kCAAA,2BAAA;;;gCAAA,kBAAA,aAAQC,mBAAAA,MAAMC,wBAAAA;gCACjB,IAAI,CAACA,WAAW,OAAA;gCAChB,IAAMC,SAASD;gCACfC,OAAOC,OAAO,CAAC,SAAC3B,GAAG4B;2CAAQP,KAAKQ,IAAI,CAAC;wCAACL;wCAAMI;wCAAK5B,EAAEC,SAAS;wCAAED,EAAEE,OAAO;qCAAC;;gCACxEoB,MAAMQ,GAAG,CACPN,MACAE,OAAO3B,GAAG,CAAC,SAACC;2CAAMA,EAAEG,IAAI;;4BAE5B;4BARA,IAAK,YAA6BiB,2BAA7B,6BAAA,QAAA,yBAAA;;4BAAA;4BAAA;;;qCAAA,6BAAA;oCAAA;;;oCAAA;0CAAA;;;;wBASL,IAAIC,KAAKL,MAAM,KAAK,GAAG;;;wBACvB;;4BAAMV,GAAGW,QAAQ,CAAC,+GAA+GI;;;wBAAjI;wBACA,uFAAuF;wBACvF,6EAA6E;wBAC7EU,IAAAA,yBAAc,EAACzB,IAAIgB;;;;;;QACrB;;IACAU,gBAAAA,SAAAA,eAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKnD,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9DoD,WAAAA,SAAAA,UAAUxB,GAAG;QACX,IAAMyB,IAAIC,IAAAA,qBAAW,EAAC1B;QACtB,IAAI,CAACyB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,IAAME,WAAWF,EAAEG,QAAQ,KAAK,WAAWC,IAAAA,yBAAa,EAACJ,EAAEK,KAAK,IAAI5C;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,IAAM6C,eAAeN,EAAE/C,WAAW,KAAKQ,YAAY,AAAC,GAAmBuC,OAAjBO,sBAAa,EAAC,KAAiB,OAAdP,EAAE/C,WAAW,IAAKsD,sBAAa;QACtG,OAAO,AAAC,SAAsBP,OAAdA,EAAEG,QAAQ,EAAC,KAAcG,OAAXN,EAAEK,KAAK,EAAC,KAAkBH,OAAfI,cAA4D,OAA7CJ,aAAazC,YAAY,AAAC,IAAY,OAATyC,YAAa;IACpG;AACF"}

package/dist/cjs/features/fences.js

@@ -1,8 +1,5 @@
-// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column
-// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see
-// test/unit/features/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
-// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the
-// opener's, holding nothing but trailing spaces after the run.
+// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column 0, no indent
+// allowance (diverges from CommonMark's up-to-3-space rule). Closer: same char, length >= opener's, trailing spaces only.
 "use strict";
 Object.defineProperty(exports, "__esModule", {
     value: true
@@ -104,10 +101,8 @@ function commentTracker() {
         }
     };
 }
-// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line
-// starting with an open or close tag of one of these, at column 0 (leading whitespace/`>`
-// blockquote markers tolerated), opens a block that swallows following lines -- tags, links,
-// and comments alike -- until a blank line closes it.
+// CommonMark's HTML-block type-6 tag list (fixed by the spec). A line opening or closing one of these at
+// column 0 (leading whitespace/`>` tolerated) swallows following lines -- tags, links, comments alike -- until a blank line.
 var HTML_BLOCK_TAGS = new Set([
     'address',
     'article',
@@ -213,7 +208,7 @@ function maskRegions(body) {
         }
         var uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;
         if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line
-        // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the
+        // Indented or blockquoted HTML blocks swallow their content in Obsidian too, so the
         // opener test runs after stripping leading whitespace and > markers.
         var stripped = uncommented.replace(/^[ \t>]*/, '');
         if (stripped[0] === '<') {

package/dist/cjs/features/fences.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column\n// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see\n// test/unit/features/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself\n// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the\n// opener's, holding nothing but trailing spaces after the run.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can\n// interleave it with their own per-line state. In isolation an unclosed <!-- stays open forever\n// (mask() has no notion of a line boundary that should kill it); maskRegions() below is what\n// calls close() at the point an unclosed comment actually dies -- a blank line at paragraph\n// level, or its enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line\n// starting with an open or close tag of one of these, at column 0 (leading whitespace/`>`\n// blockquote markers tolerated), opens a block that swallows following lines -- tags, links,\n// and comments alike -- until a blank line closes it.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines\n// preserved, so line numbers and offsets survive) plus inline code spans masked. The single\n// region masker for tags/links: fence and HTML-block delimiter+content lines are opaque, an\n// unclosed comment dies at its containing block's end (or the next blank line at paragraph\n// level -- never silently to end of file), and a closed comment masks exactly its span.\n// Bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N -- a\n// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).\n// Masked with spaces so column positions and tag-boundary whitespace are unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["commentTracker","fenceTracker","maskCodeSpans","maskRegions","BACKTICK_OPEN","TILDE_OPEN","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,+FAA+F;AAC/F,8FAA8F;AAC9F,wGAAwG;AACxG,yFAAyF;AACzF,+DAA+D;;;;;;;;;;;;QA0D/CA;eAAAA;;QAhDAC;eAAAA;;QAyOAC;eAAAA;;QA/DAC;eAAAA;;;AA7KhB,IAAMC,gBAAgB;AACtB,IAAMC,aAAa;AAEZ,SAASJ;IACd,IAAIK,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAAA,SAAAA,KAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,IAAMK,KAAKP,cAAcQ,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,IAAMC,KAAKV,WAAWO,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,IAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,IAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAaO,SAASN;IACd,IAAImB,YAAY;IAChB,OAAO;QACLC,MAAAA,SAAAA,KAAKV,IAAY;YACf,IAAIW,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;gBACtB,IAAIK,WAAW;oBACb,IAAMI,MAAMb,KAAKc,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAACf,KAAKI,MAAM,GAAGQ;wBAChCA,IAAIZ,KAAKI,MAAM;oBACjB,OAAO;wBACLO,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,IAAMO,QAAQhB,KAAKc,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOX,KAAKiB,KAAK,CAACL;oBAClBA,IAAIZ,KAAKI,MAAM;gBACjB,OAAO;oBACLO,OAAO,AAAC,GAAuB,OAArBX,KAAKiB,KAAK,CAACL,GAAGI,QAAO;oBAC/BJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO,OAAAA,SAAAA;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,8FAA8F;AAC9F,0FAA0F;AAC1F,6FAA6F;AAC7F,sDAAsD;AACtD,IAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,IAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,IAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,IAAMC,qBAAqB;AAC3B,IAAMC,gBAAgB,aAAa,wFAAwF;AAQpH,SAAS/B,YAAYgC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,IAAME,QAAQpC;IACd,IAAMqC,UAAUtC;IAChB,IAAIuC,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,SAAChC;QACJ,IAAI6B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC1B,OAAO;oBAC7B6B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIf,MAAM,CAACf,KAAKI,MAAM;YAC/B;YACA,IAAIoB,cAAcE,IAAI,CAAC1B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC6B,cAAc;gBACd,OAAO7B;YACT;YACA,OAAO,IAAIe,MAAM,CAACf,KAAKI,MAAM;QAC/B;QACA,IAAI,CAACwB,QAAQnB,SAAS,EAAE;YACtB,IAAMwB,UAAUN,MAAM5B,IAAI,CAACC;YAC3B,IAAIiC,WAAWN,MAAM/B,OAAO,EAAE,OAAO,IAAImB,MAAM,CAACf,KAAKI,MAAM;QAC7D;QACA,IAAM8B,cAAcN,QAAQnB,SAAS,IAAIT,KAAKG,QAAQ,CAAC,UAAUyB,QAAQlB,IAAI,CAACV,QAAQA;QACtF,IAAI4B,QAAQnB,SAAS,IAAIe,cAAcE,IAAI,CAAC1B,OAAO4B,QAAQV,KAAK,IAAI,+CAA+C;QACnH,sFAAsF;QACtF,qEAAqE;QACrE,IAAMiB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,IAAME,YAAYd,mBAAmBrB,IAAI,CAACiC;YAC1C,IAAIE,WAAW;gBACb,IAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,IAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBnB,cAAcoB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAAChB,kBAAkBI,IAAI,CAACS,SAASlB,KAAK,CAACoB,SAAS,CAAC,EAAE,CAACjC,MAAM,IAAI;wBAChEyB,cAAc;wBACdC,iBAAiBR;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAACf,KAAKI,MAAM;gBAC/B;gBACA,IAAIe,gBAAgBsB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAId,MAAM,CAACf,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAO8B,YAAY/B,QAAQ,CAAC,OAAOX,cAAc0C,eAAeA;IAClE,GACCQ,IAAI,CAAC;AACV;AAKO,SAASlD,cAAcQ,IAAY;IACxC,IAAIW,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACY,EAAE,KAAK,KAAK;YACnBD,OAAOX,IAAI,CAACY,EAAE;YACdA;YACA;QACF;QACA,IAAI+B,IAAI/B;QACR,MAAOZ,IAAI,CAAC2C,EAAE,KAAK,IAAKA;QACxB,IAAMC,IAAID,IAAI/B;QACd,IAAIiC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAI7C,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAAC6C,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAItC,IAAIsC;YACR,MAAO7C,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIsC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAWxC;gBACX;YACF;YACAsC,IAAItC;QACN;QACA,IAAIuC,cAAc,GAAG;YACnBnC,OAAO,IAAII,MAAM,CAACgC,WAAWnC;YAC7BA,IAAImC;QACN,OAAO;YACLpC,OAAOX,KAAKiB,KAAK,CAACL,GAAG+B;YACrB/B,IAAI+B;QACN;IACF;IACA,OAAOhC;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/fences.ts"],"sourcesContent":["// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column 0, no indent\n// allowance (diverges from CommonMark's up-to-3-space rule). Closer: same char, length >= opener's, trailing spaces only.\n\nexport interface FenceTracker {\n  feed(line: string): boolean; // true iff this line is a fence delimiter (open or close)\n  readonly inFence: boolean;\n}\n\nconst BACKTICK_OPEN = /^(`{3,})(.*)$/;\nconst TILDE_OPEN = /^(~{3,})(.*)$/;\n\nexport function fenceTracker(): FenceTracker {\n  let inFence = false;\n  let fenceChar = '';\n  let fenceLen = 0;\n  return {\n    feed(line: string): boolean {\n      if (!inFence) {\n        const bt = BACKTICK_OPEN.exec(line);\n        if (bt && !bt[2].includes('`')) {\n          inFence = true;\n          fenceChar = '`';\n          fenceLen = bt[1].length;\n          return true;\n        }\n        const td = TILDE_OPEN.exec(line);\n        if (td) {\n          inFence = true;\n          fenceChar = '~';\n          fenceLen = td[1].length;\n          return true;\n        }\n        return false;\n      }\n      const run = fenceChar === '`' ? /^(`+)(.*)$/ : /^(~+)(.*)$/;\n      const m = run.exec(line);\n      if (m && m[1].length >= fenceLen && m[2].trim() === '') {\n        inFence = false;\n        return true;\n      }\n      return false;\n    },\n    get inFence() {\n      return inFence;\n    },\n  };\n}\n\n// Masks <!-- ... --> spans that may cross line boundaries, one line at a time so callers can interleave their own per-line state.\n// An unclosed comment stays open forever unless closed externally; maskRegions() below calls close() at a paragraph blank line or the enclosing HTML block's end.\nexport interface CommentTracker {\n  mask(line: string): string;\n  close(): void; // force-ends an unclosed comment (paragraph blank line, or its HTML block ending)\n  readonly inComment: boolean;\n}\n\nexport function commentTracker(): CommentTracker {\n  let inComment = false;\n  return {\n    mask(line: string): string {\n      let out = '';\n      let i = 0;\n      while (i < line.length) {\n        if (inComment) {\n          const end = line.indexOf('-->', i);\n          if (end === -1) {\n            out += ' '.repeat(line.length - i);\n            i = line.length;\n          } else {\n            out += ' '.repeat(end + 3 - i);\n            i = end + 3;\n            inComment = false;\n          }\n          continue;\n        }\n        const start = line.indexOf('<!--', i);\n        if (start === -1) {\n          out += line.slice(i);\n          i = line.length;\n        } else {\n          out += `${line.slice(i, start)}    `;\n          i = start + 4;\n          inComment = true;\n        }\n      }\n      return out;\n    },\n    close() {\n      inComment = false;\n    },\n    get inComment() {\n      return inComment;\n    },\n  };\n}\n\n// CommonMark's HTML-block type-6 tag list (fixed by the spec). A line opening or closing one of these at\n// column 0 (leading whitespace/`>` tolerated) swallows following lines -- tags, links, comments alike -- until a blank line.\nconst HTML_BLOCK_TAGS = new Set([\n  'address',\n  'article',\n  'aside',\n  'base',\n  'basefont',\n  'blockquote',\n  'body',\n  'caption',\n  'center',\n  'col',\n  'colgroup',\n  'dd',\n  'details',\n  'dialog',\n  'dir',\n  'div',\n  'dl',\n  'dt',\n  'fieldset',\n  'figcaption',\n  'figure',\n  'footer',\n  'form',\n  'frame',\n  'frameset',\n  'h1',\n  'h2',\n  'h3',\n  'h4',\n  'h5',\n  'h6',\n  'head',\n  'header',\n  'hr',\n  'html',\n  'iframe',\n  'legend',\n  'li',\n  'link',\n  'main',\n  'menu',\n  'menuitem',\n  'nav',\n  'noframes',\n  'ol',\n  'optgroup',\n  'option',\n  'p',\n  'param',\n  'search',\n  'section',\n  'summary',\n  'table',\n  'tbody',\n  'td',\n  'tfoot',\n  'th',\n  'thead',\n  'title',\n  'tr',\n  'track',\n  'ul',\n]);\n// Type-1 blocks (script/pre/style/textarea): closes on the line holding the matching end tag,\n// not on a blank line, and that line is the last one swallowed.\nconst HTML_PRE_TAGS = new Set(['script', 'pre', 'style', 'textarea']);\nconst HTML_PRE_CLOSE_RE = /<\\/(?:script|pre|style|textarea)>/i; // any of the four closers ends the block, not only the tag that opened it\n// An opening or closing tag at column 0, tag name captured for the lookups above.\nconst HTML_BLOCK_OPEN_RE = /^<\\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \\t]|\\/?>|$)/;\nconst BLANK_LINE_RE = /^[ \\t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block\n\n// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines preserved, so line numbers and offsets survive), inline code spans masked too.\n// An unclosed comment dies at its block's end or the next paragraph blank line, never silently to end of file; bodies without a backtick/tilde/`<` fast-path out untouched.\nexport function maskRegions(body: string): string {\n  if (!/[`~<]/.test(body)) return body;\n  const fence = fenceTracker();\n  const comment = commentTracker();\n  let inHtmlBlock = false;\n  let htmlBlockClose: RegExp | null = null; // set while inside a type-1 (script/pre/style/textarea) block\n  return body\n    .split('\\n')\n    .map((line) => {\n      if (inHtmlBlock) {\n        if (htmlBlockClose) {\n          if (htmlBlockClose.test(line)) {\n            inHtmlBlock = false;\n            htmlBlockClose = null;\n          }\n          return ' '.repeat(line.length);\n        }\n        if (BLANK_LINE_RE.test(line)) {\n          // Block content never feeds the comment tracker, so an inner <!-- was only ever\n          // masked text; nothing to close here.\n          inHtmlBlock = false;\n          return line;\n        }\n        return ' '.repeat(line.length);\n      }\n      if (!comment.inComment) {\n        const isDelim = fence.feed(line);\n        if (isDelim || fence.inFence) return ' '.repeat(line.length);\n      }\n      const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;\n      if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line\n      // Indented or blockquoted HTML blocks swallow their content in Obsidian too, so the\n      // opener test runs after stripping leading whitespace and > markers.\n      const stripped = uncommented.replace(/^[ \\t>]*/, '');\n      if (stripped[0] === '<') {\n        const openMatch = HTML_BLOCK_OPEN_RE.exec(stripped);\n        if (openMatch) {\n          const tagName = openMatch[1].toLowerCase();\n          const isClosingTag = stripped[1] === '/';\n          if (!isClosingTag && HTML_PRE_TAGS.has(tagName)) {\n            // The end condition can be met on the opening line itself: <script>x</script>\n            // is a one-line block and the next line parses (Obsidian-verified).\n            if (!HTML_PRE_CLOSE_RE.test(stripped.slice(openMatch[0].length))) {\n              inHtmlBlock = true;\n              htmlBlockClose = HTML_PRE_CLOSE_RE;\n            }\n            return ' '.repeat(line.length);\n          }\n          if (HTML_BLOCK_TAGS.has(tagName)) {\n            inHtmlBlock = true;\n            return ' '.repeat(line.length);\n          }\n        }\n      }\n      // Inline code spans hide their content too: `[[x]]` in prose is not a link.\n      return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;\n    })\n    .join('\\n');\n}\n\n// A code span opens on a run of N backticks and closes at the next run of exactly N (CommonMark code spans);\n// a shorter/longer run in between is literal text. Masked with spaces so column positions stay unaffected.\nexport function maskCodeSpans(line: string): string {\n  let out = '';\n  let i = 0;\n  while (i < line.length) {\n    if (line[i] !== '`') {\n      out += line[i];\n      i++;\n      continue;\n    }\n    let j = i;\n    while (line[j] === '`') j++;\n    const n = j - i;\n    let k = j;\n    let closeStart = -1;\n    let closeEnd = -1;\n    while (k < line.length) {\n      if (line[k] !== '`') {\n        k++;\n        continue;\n      }\n      let m = k;\n      while (line[m] === '`') m++;\n      if (m - k === n) {\n        closeStart = k;\n        closeEnd = m;\n        break;\n      }\n      k = m;\n    }\n    if (closeStart >= 0) {\n      out += ' '.repeat(closeEnd - i);\n      i = closeEnd;\n    } else {\n      out += line.slice(i, j);\n      i = j;\n    }\n  }\n  return out;\n}\n"],"names":["commentTracker","fenceTracker","maskCodeSpans","maskRegions","BACKTICK_OPEN","TILDE_OPEN","inFence","fenceChar","fenceLen","feed","line","bt","exec","includes","length","td","run","m","trim","inComment","mask","out","i","end","indexOf","repeat","start","slice","close","HTML_BLOCK_TAGS","Set","HTML_PRE_TAGS","HTML_PRE_CLOSE_RE","HTML_BLOCK_OPEN_RE","BLANK_LINE_RE","body","test","fence","comment","inHtmlBlock","htmlBlockClose","split","map","isDelim","uncommented","stripped","replace","openMatch","tagName","toLowerCase","isClosingTag","has","join","j","n","k","closeStart","closeEnd"],"mappings":"AAAA,4GAA4G;AAC5G,0HAA0H;;;;;;;;;;;;QAuD1GA;eAAAA;;QA7CAC;eAAAA;;QA+NAC;eAAAA;;QA9DAC;eAAAA;;;AApKhB,IAAMC,gBAAgB;AACtB,IAAMC,aAAa;AAEZ,SAASJ;IACd,IAAIK,UAAU;IACd,IAAIC,YAAY;IAChB,IAAIC,WAAW;IACf,OAAO;QACLC,MAAAA,SAAAA,KAAKC,IAAY;YACf,IAAI,CAACJ,SAAS;gBACZ,IAAMK,KAAKP,cAAcQ,IAAI,CAACF;gBAC9B,IAAIC,MAAM,CAACA,EAAE,CAAC,EAAE,CAACE,QAAQ,CAAC,MAAM;oBAC9BP,UAAU;oBACVC,YAAY;oBACZC,WAAWG,EAAE,CAAC,EAAE,CAACG,MAAM;oBACvB,OAAO;gBACT;gBACA,IAAMC,KAAKV,WAAWO,IAAI,CAACF;gBAC3B,IAAIK,IAAI;oBACNT,UAAU;oBACVC,YAAY;oBACZC,WAAWO,EAAE,CAAC,EAAE,CAACD,MAAM;oBACvB,OAAO;gBACT;gBACA,OAAO;YACT;YACA,IAAME,MAAMT,cAAc,MAAM,eAAe;YAC/C,IAAMU,IAAID,IAAIJ,IAAI,CAACF;YACnB,IAAIO,KAAKA,CAAC,CAAC,EAAE,CAACH,MAAM,IAAIN,YAAYS,CAAC,CAAC,EAAE,CAACC,IAAI,OAAO,IAAI;gBACtDZ,UAAU;gBACV,OAAO;YACT;YACA,OAAO;QACT;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAUO,SAASN;IACd,IAAImB,YAAY;IAChB,OAAO;QACLC,MAAAA,SAAAA,KAAKV,IAAY;YACf,IAAIW,MAAM;YACV,IAAIC,IAAI;YACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;gBACtB,IAAIK,WAAW;oBACb,IAAMI,MAAMb,KAAKc,OAAO,CAAC,OAAOF;oBAChC,IAAIC,QAAQ,CAAC,GAAG;wBACdF,OAAO,IAAII,MAAM,CAACf,KAAKI,MAAM,GAAGQ;wBAChCA,IAAIZ,KAAKI,MAAM;oBACjB,OAAO;wBACLO,OAAO,IAAII,MAAM,CAACF,MAAM,IAAID;wBAC5BA,IAAIC,MAAM;wBACVJ,YAAY;oBACd;oBACA;gBACF;gBACA,IAAMO,QAAQhB,KAAKc,OAAO,CAAC,QAAQF;gBACnC,IAAII,UAAU,CAAC,GAAG;oBAChBL,OAAOX,KAAKiB,KAAK,CAACL;oBAClBA,IAAIZ,KAAKI,MAAM;gBACjB,OAAO;oBACLO,OAAO,AAAC,GAAuB,OAArBX,KAAKiB,KAAK,CAACL,GAAGI,QAAO;oBAC/BJ,IAAII,QAAQ;oBACZP,YAAY;gBACd;YACF;YACA,OAAOE;QACT;QACAO,OAAAA,SAAAA;YACET,YAAY;QACd;QACA,IAAIA;YACF,OAAOA;QACT;IACF;AACF;AAEA,yGAAyG;AACzG,6HAA6H;AAC7H,IAAMU,kBAAkB,IAAIC,IAAI;IAC9B;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;IACA;CACD;AACD,8FAA8F;AAC9F,gEAAgE;AAChE,IAAMC,gBAAgB,IAAID,IAAI;IAAC;IAAU;IAAO;IAAS;CAAW;AACpE,IAAME,oBAAoB,sCAAsC,0EAA0E;AAC1I,kFAAkF;AAClF,IAAMC,qBAAqB;AAC3B,IAAMC,gBAAgB,aAAa,wFAAwF;AAIpH,SAAS/B,YAAYgC,IAAY;IACtC,IAAI,CAAC,QAAQC,IAAI,CAACD,OAAO,OAAOA;IAChC,IAAME,QAAQpC;IACd,IAAMqC,UAAUtC;IAChB,IAAIuC,cAAc;IAClB,IAAIC,iBAAgC,MAAM,8DAA8D;IACxG,OAAOL,KACJM,KAAK,CAAC,MACNC,GAAG,CAAC,SAAChC;QACJ,IAAI6B,aAAa;YACf,IAAIC,gBAAgB;gBAClB,IAAIA,eAAeJ,IAAI,CAAC1B,OAAO;oBAC7B6B,cAAc;oBACdC,iBAAiB;gBACnB;gBACA,OAAO,IAAIf,MAAM,CAACf,KAAKI,MAAM;YAC/B;YACA,IAAIoB,cAAcE,IAAI,CAAC1B,OAAO;gBAC5B,gFAAgF;gBAChF,sCAAsC;gBACtC6B,cAAc;gBACd,OAAO7B;YACT;YACA,OAAO,IAAIe,MAAM,CAACf,KAAKI,MAAM;QAC/B;QACA,IAAI,CAACwB,QAAQnB,SAAS,EAAE;YACtB,IAAMwB,UAAUN,MAAM5B,IAAI,CAACC;YAC3B,IAAIiC,WAAWN,MAAM/B,OAAO,EAAE,OAAO,IAAImB,MAAM,CAACf,KAAKI,MAAM;QAC7D;QACA,IAAM8B,cAAcN,QAAQnB,SAAS,IAAIT,KAAKG,QAAQ,CAAC,UAAUyB,QAAQlB,IAAI,CAACV,QAAQA;QACtF,IAAI4B,QAAQnB,SAAS,IAAIe,cAAcE,IAAI,CAAC1B,OAAO4B,QAAQV,KAAK,IAAI,+CAA+C;QACnH,oFAAoF;QACpF,qEAAqE;QACrE,IAAMiB,WAAWD,YAAYE,OAAO,CAAC,YAAY;QACjD,IAAID,QAAQ,CAAC,EAAE,KAAK,KAAK;YACvB,IAAME,YAAYd,mBAAmBrB,IAAI,CAACiC;YAC1C,IAAIE,WAAW;gBACb,IAAMC,UAAUD,SAAS,CAAC,EAAE,CAACE,WAAW;gBACxC,IAAMC,eAAeL,QAAQ,CAAC,EAAE,KAAK;gBACrC,IAAI,CAACK,gBAAgBnB,cAAcoB,GAAG,CAACH,UAAU;oBAC/C,8EAA8E;oBAC9E,oEAAoE;oBACpE,IAAI,CAAChB,kBAAkBI,IAAI,CAACS,SAASlB,KAAK,CAACoB,SAAS,CAAC,EAAE,CAACjC,MAAM,IAAI;wBAChEyB,cAAc;wBACdC,iBAAiBR;oBACnB;oBACA,OAAO,IAAIP,MAAM,CAACf,KAAKI,MAAM;gBAC/B;gBACA,IAAIe,gBAAgBsB,GAAG,CAACH,UAAU;oBAChCT,cAAc;oBACd,OAAO,IAAId,MAAM,CAACf,KAAKI,MAAM;gBAC/B;YACF;QACF;QACA,4EAA4E;QAC5E,OAAO8B,YAAY/B,QAAQ,CAAC,OAAOX,cAAc0C,eAAeA;IAClE,GACCQ,IAAI,CAAC;AACV;AAIO,SAASlD,cAAcQ,IAAY;IACxC,IAAIW,MAAM;IACV,IAAIC,IAAI;IACR,MAAOA,IAAIZ,KAAKI,MAAM,CAAE;QACtB,IAAIJ,IAAI,CAACY,EAAE,KAAK,KAAK;YACnBD,OAAOX,IAAI,CAACY,EAAE;YACdA;YACA;QACF;QACA,IAAI+B,IAAI/B;QACR,MAAOZ,IAAI,CAAC2C,EAAE,KAAK,IAAKA;QACxB,IAAMC,IAAID,IAAI/B;QACd,IAAIiC,IAAIF;QACR,IAAIG,aAAa,CAAC;QAClB,IAAIC,WAAW,CAAC;QAChB,MAAOF,IAAI7C,KAAKI,MAAM,CAAE;YACtB,IAAIJ,IAAI,CAAC6C,EAAE,KAAK,KAAK;gBACnBA;gBACA;YACF;YACA,IAAItC,IAAIsC;YACR,MAAO7C,IAAI,CAACO,EAAE,KAAK,IAAKA;YACxB,IAAIA,IAAIsC,MAAMD,GAAG;gBACfE,aAAaD;gBACbE,WAAWxC;gBACX;YACF;YACAsC,IAAItC;QACN;QACA,IAAIuC,cAAc,GAAG;YACnBnC,OAAO,IAAII,MAAM,CAACgC,WAAWnC;YAC7BA,IAAImC;QACN,OAAO;YACLpC,OAAOX,KAAKiB,KAAK,CAACL,GAAG+B;YACrB/B,IAAI+B;QACN;IACF;IACA,OAAOhC;AACT"}

package/dist/cjs/features/links.js

@@ -263,22 +263,18 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Map" || n === "Set") return Array.from(n);
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
-// links(src, target, target_base, dst, embed): target as written, target_base its baseKey
-// (indexed, for the incremental resolve below), dst the resolved path or NULL (a
-// queryable dead link), embed whether it's `![[x]]`/`![](x.md)` rather than `[[x]]`/`[](x.md)`
-// -- Obsidian's grain: a target both linked and embedded in the same note is two rows.
-// Wikilinks ([[target]], [[target#anchor|alias]], embeds), internal markdown links, and
-// frontmatter values that are exactly a wikilink ("[[X]]"; mid-string and ![[...]] forms are
-// not links there -- Obsidian's rule, probe-verified).
-// One rule for [[inner]] text wherever it appears: alias stripped after |, a leading #
-// keeps the anchor as the target (a same-note self-link needs a heading name), otherwise
-// the anchor splits off. Null = not a link.
+// links(src, target, target_base, dst, embed): target as written, target_base its indexed baseKey, dst the resolved path or NULL (a queryable dead link).
+// embed distinguishes `![[x]]`/`![](x.md)` from `[[x]]`/`[](x.md)` -- Obsidian's grain: a target both linked and embedded in one note is two rows.
+// Alias stripped after |; a leading # keeps the anchor as the target (self-link needs a
+// heading name), else the anchor splits off. Null = not a link.
 function parseWikilinkInner(inner) {
     var beforeAlias = inner.split('|')[0].trim();
     if (beforeAlias.startsWith('#')) return beforeAlias.length > 1 ? beforeAlias : null;
     var target = beforeAlias.split('#')[0].trim();
     return target || null;
 }
+// A frontmatter value counts only when it is exactly a wikilink ("[[X]]") -- mid-string and
+// ![[...]] forms don't count there, per Obsidian's rule (probe-verified).
 function extract(_raw, rawBody, _search, data) {
     var body = /\[\[|\]\(/.test(rawBody) ? (0, _fencests.maskRegions)(rawBody) : rawBody;
     var seen = new Set();
@@ -318,12 +314,8 @@ function extract(_raw, rawBody, _search, data) {
     }
     var _iteratorNormalCompletion1 = true, _didIteratorError1 = false, _iteratorError1 = undefined;
     try {
-        // Any internal destination, not only .md-suffixed: Obsidian gives markdown links full
-        // linkpath resolution, so \](Zektor) resolves like [[Zektor]] and \](#anchor) is a
-        // self-edge. External URLs (a scheme anywhere survives the malformed double-paren case)
-        // and titled links (dest stops at whitespace) are skipped.
-        // A space in the destination is only valid ahead of a quoted title; a domain-shaped first
-        // segment (www.example.com/...) is external even without a scheme.
+        // Any internal destination, not only .md-suffixed: \](Zektor) resolves like [[Zektor]] and
+        // \](#anchor) is a self-edge. External URLs and titled links (dest stops at whitespace) skip.
         for(var _iterator1 = body.matchAll(/(!)?\[[^\]]*\]\(((?:[^()\s]|\([^()\s]*\))+)(?:\s+(?:"[^)]*"|'[^)]*'))?\)/g)[Symbol.iterator](), _step1; !(_iteratorNormalCompletion1 = (_step1 = _iterator1.next()).done); _iteratorNormalCompletion1 = true){
             var m1 = _step1.value;
             var dest = m1[2].trim();
@@ -401,9 +393,8 @@ function cleanTarget(target) {
 function baseKey(path) {
     return _posix.default.basename(path).replace(/\.md$/i, '').toLowerCase();
 }
-// Obsidian-style: exact relative path (with/without .md), path relative to the linking
-// note's directory, then basename match (shortest path wins on ties -- verified against a
-// cold-loaded Obsidian vault on 6+ real collision pairs; byBase's lists are pre-sorted that way).
+// Obsidian-style: exact relative path (with/without .md), path relative to the linking note's
+// directory, then basename match (shortest path wins ties, verified against a cold-loaded vault).
 function resolveTarget(src, target, pathSet, byBase) {
     // [[#Heading]]: a same-note anchor, always the note itself -- never depends on other files.
     if (target.startsWith('#')) return src;
@@ -460,9 +451,8 @@ function buildByBase(files) {
     }
     return byBase;
 }
-// resolveTarget applied to `rows`, returning just the rows whose dst actually changed -- the
-// caller writes those in one runBatch call, keyed on (src, target) so a link/embed sibling
-// pair sharing one resolution rewrites both.
+// resolveTarget applied to `rows`, returning only the rows whose dst changed, so the caller
+// can write them in one runBatch call.
 function resolveRows(rows, pathSet, byBase) {
     var changed = [];
     var _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
@@ -611,9 +601,8 @@ function selectIn(db, column, keys) {
         });
     })();
 }
-// Re-resolve only what this reconcile could have touched: re-stored sources, links whose
-// target basename matches an added/vanished file, and links whose dst just vanished. All
-// three are indexed lookups, so this never reads the whole table.
+// Re-resolves only what this reconcile could have touched: re-stored sources, links whose
+// target basename matches an added/vanished file, and links whose dst vanished -- all indexed.
 function resolveIncremental(db, delta) {
     return _async_to_generator(function() {
         var pathSet, byBase, changedBasenames, _iteratorNormalCompletion, _didIteratorError, _iteratorError, _iterator, _step, p, _iteratorNormalCompletion1, _didIteratorError1, _iteratorError1, _iterator1, _step1, p1, candidates, collect, changed;
@@ -764,10 +753,8 @@ function linkEdges(db) {
         });
     })();
 }
-// remove()/store() have already deleted rows by the time afterReconcile runs, so this records
-// whether any reconcile pass deleted at least one link row -- whether a file vanished, or a
-// reparse's stale rows shrank. dstChanged alone misses the reparsed-to-zero-links case (no
-// surviving rows to re-resolve). Keyed on the delta object, so the state dies with the reconcile.
+// Records whether remove()/store() deleted any link row, since afterReconcile runs after both
+// and dstChanged alone misses the reparsed-to-zero-links case. Keyed on delta so state dies with it.
 var removedWithLinks = new WeakMap();
 var links = {
     name: 'links',
@@ -802,10 +789,8 @@ var links = {
         })();
     },
     extract: extract,
-    remove: // Reparsed docs are diffed in store() below instead of wiped here: deleting and re-inserting
-    // resets dst to NULL, which made every touch-only reparse read as an edge change and
-    // recompute PageRank -- measured as most of the remaining update cost. Only genuinely
-    // vanished files clear here.
+    remove: // Reparsed docs are diffed in store() instead of wiped here: delete-and-reinsert resets dst
+    // to NULL, making every touch-only reparse look like an edge change and recompute PageRank.
     function remove(db, paths, delta) {
         return _async_to_generator(function() {
             var vanishedSet, vanished, existingRows;
@@ -893,10 +878,8 @@ var links = {
                         removedWithLinks.set(delta, true);
                         _state.label = 3;
                     case 3:
-                        // Upsert preserves dst on surviving rows, so an unchanged link resolves to the same value
-                        // and reports no change; only genuinely new rows start at NULL. A same-note anchor's dst is
-                        // always its own src, never another file's basename, so it gets no target_base -- SQL's
-                        // `IN` never matches NULL, keeping it out of resolveIncremental's target_base lookup.
+                        // Upsert preserves dst on surviving rows; only new rows start at NULL. A same-note anchor
+                        // gets no target_base, so SQL's `IN` excludes it from resolveIncremental's lookup.
                         upsertRows = [];
                         _iteratorNormalCompletion = true, _didIteratorError = false, _iteratorError = undefined;
                         try {