sensemaking 0.18.4 → 0.19.1

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL, re-deriving chunk text from the files through the\n// same parse + chunker that stored the rows.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    let chunks: Chunk[];\n    try {\n      // presets/embed are irrelevant here -- re-deriving chunk text for a doc that already\n      // has embeddings rows means the tree had an embedding model at reconcile time.\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality,\n// and nearest-neighbour search always returns a neighbour however far away.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["join","SenseError","embed","progress","parseFile","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","store","cfg","baseDir","provider","dirty","vectors","pending","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","jobs","chunkIdxs","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","idx","text","map","j","report","i","batchCap","batch","slice","embedDocuments","writeVectors","job","q","Int8Array","round","vector","Buffer","from","buffer","tick","finish","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","candidates","hasEmbedding","hasVector","similarNotes","_cfg","opts","similar"],"mappings":"AAAA,SAASA,IAAI,QAAQ,YAAY;AAEjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAASC,UAAU,QAAQ,aAAa;AAExC,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,qFAAqF;AACrF,6CAA6C;AAC7C,OAAO,eAAeE,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAAe;IAC3E,MAAMC,WAAW,MAAMnB,YAAYiB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQ,MAAMJ,MAAMK,OAAO,CAACC,OAAO;IACzC,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACJ,MAAMK,UAAU,IAAIV,OAAQ;QACtC,IAAIW;QACJ,IAAI;YACF,qFAAqF;YACrF,+EAA+E;YAC/EA,SAASvC,UAAU;gBAAEwC,SAASP;gBAAMQ,SAAS7C,KAAKwB,SAASa;gBAAOS,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAE/C,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEqB,KAAK2B,GAAG,CAACC,SAAS,CAACjD,KAAK;QAClK,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAMkD,OAAOV,UAAW,IAAIC,MAAM,CAACS,IAAI,EAAEX,KAAKH,IAAI,CAAC;YAAED;YAAME,OAAOa;YAAKC,MAAMV,MAAM,CAACS,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAMhD,iBACJiB,OACAG,UACAgB,KAAKa,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;IAGxB,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAMG,SAASrD,SAAS,oBAAoBsC,KAAKZ,MAAM;IACvD,IAAK,IAAI4B,IAAI,GAAGA,IAAIhB,KAAKZ,MAAM,EAAE4B,KAAKhC,SAASiC,QAAQ,CAAE;QACvD,MAAMC,QAAQlB,KAAKmB,KAAK,CAACH,GAAGA,IAAIhC,SAASiC,QAAQ;QACjD,MAAM/B,UAAU,MAAMF,SAASoC,cAAc,CAACF,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEF,IAAI;QACrE,MAAM/B,MAAMK,OAAO,CAACmC,YAAY,CAC9BH,MAAML,GAAG,CAAC,CAACS,KAAKR;YACd,MAAM,EAAE3C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQmB,OAAO,CAAC4B,EAAE,EAAEzB,WAAW;YACpD,MAAMkC,IAAI,IAAIC,UAAUnC;YACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKiD,CAAC,CAACjD,EAAE,GAAGC,KAAKkD,KAAK,CAACtD,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEmB,MAAM0B,IAAI1B,IAAI;gBAAEE,OAAOwB,IAAIxB,KAAK;gBAAErB;gBAAOiD,QAAQC,OAAOC,IAAI,CAACL,EAAEM,MAAM;YAAE;QAClF;QAEFd,OAAOe,IAAI,CAACvD,KAAKe,GAAG,CAAC0B,IAAIhC,SAASiC,QAAQ,EAAEjB,KAAKZ,MAAM;IACzD;IACA2B,OAAOgB,MAAM;AACf;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,4EAA4E;AAC5E,OAAO,eAAeC,mBAAmBnD,KAAY,EAAEC,GAAW,EAAEmD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAOvGF;IANd,MAAMlD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIvB,WAAW,eAAe;IAClD,MAAMoB,aAAaC,OAAOC,KAAKC;IAE/B,MAAMC,WAAW,MAAMnB,YAAYiB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IACpD,MAAM2C,OAAO,EAACqB,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAI/E,IAAI,CAAC;IAClH,MAAM,EAAEY,GAAGqE,EAAE,EAAE,GAAGzE,QAAQ,MAAMiB,SAASyD,UAAU,CAAC7B,OAAOvB,WAAW;IAEtE,OAAOR,MAAMK,OAAO,CAACwD,UAAU,CAACF,IAAInD,WAAW6C,OAAOC;AACxD;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeQ,aAAa9D,KAAY,EAAEe,IAAY;IAC3D,OAAOf,MAAMK,OAAO,CAAC0D,SAAS,CAAChD;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiD,aAAahE,KAAY,EAAEiE,IAAY,EAAElD,IAAY,EAAEmD,IAAgE;IAC3I,OAAOlE,MAAMK,OAAO,CAAC8D,OAAO,CAACpD,MAAMmD;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { existsSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseFile } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the file.\nexport async function embedPending(store: Store, cfg: Config, baseDir: string): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = await store.vectors.pending();\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map<string, string[]>();\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && chunkIdxs.every((idx) => idx < texts.length)) {\n      // A stat, not a parse: a file that vanished since reconcile is skipped here even though its\n      // text comes from memory, and the next reconcile removes its rows.\n      if (!existsSync(join(baseDir, path))) continue;\n      for (const idx of chunkIdxs) jobs.push({ path, chunk: idx, text: texts[idx] });\n      continue;\n    }\n    // Reconcile ran in another process (`sense map` earlier, or a background `sense watch`), so\n    // its chunk text is gone: re-derive this one file.\n    let chunks: Chunk[];\n    try {\n      chunks = parseFile({ relPath: path, absPath: join(baseDir, path), mtimeMs: 0, ctimeMs: 0, size: 0, presets: [], embed: true }, [embed], cfg).doc.extracted.embed as Chunk[];\n    } catch {\n      continue; // vanished since reconcile; the next reconcile removes its rows\n    }\n    for (const idx of chunkIdxs) if (chunks[idx]) jobs.push({ path, chunk: idx, text: chunks[idx].text });\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // The lazy build is the one long silence a first search hits (measured 23s at\n  // 26k notes); progress makes it distinguishable from a hang.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const baseDir = (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!baseDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a config with baseDir (use loadConfig/open)');\n  await embedPending(store, cfg, baseDir);\n\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["existsSync","join","SenseError","embed","progress","parseFile","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","store","cfg","baseDir","provider","dirty","vectors","pending","length","storeDims","min","byPath","Map","row","list","get","path","push","chunk","set","textByPath","jobs","chunkIdxs","texts","every","idx","text","chunks","relPath","absPath","mtimeMs","ctimeMs","size","presets","doc","extracted","map","j","report","i","batchCap","batch","slice","embedDocuments","writeVectors","job","q","Int8Array","round","vector","Buffer","from","buffer","tick","finish","semanticCandidates","terms","fetch","allowed","match","filter","t","includes","qv","embedQuery","candidates","hasEmbedding","hasVector","similarNotes","_cfg","opts","similar"],"mappings":"AAAA,SAASA,UAAU,QAAQ,UAAU;AACrC,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,SAAS,QAAQ,mBAAmB;AAE7C,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAASC,UAAU,QAAQ,aAAa;AAExC,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,OAAO,eAAeE,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAAe;QAaxDpB;IAZnB,MAAMqB,WAAW,MAAMnB,YAAYiB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQ,MAAMJ,MAAMK,OAAO,CAACC,OAAO;IACzC,IAAIF,MAAMG,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IAEpD,MAAMsB,SAAS,IAAIC;IACnB,KAAK,MAAMC,OAAOR,MAAO;YACVM;QAAb,MAAMG,QAAOH,cAAAA,OAAOI,GAAG,CAACF,IAAIG,IAAI,eAAnBL,yBAAAA,cAAwB,EAAE;QACvCG,KAAKG,IAAI,CAACJ,IAAIK,KAAK;QACnBP,OAAOQ,GAAG,CAACN,IAAIG,IAAI,EAAEF;IACvB;IAEA,MAAMM,cAAarC,iBAAAA,cAAckB,oBAAdlB,4BAAAA,iBAAwB,IAAI6B;IAE/C,MAAMS,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACL,MAAMM,UAAU,IAAIX,OAAQ;QACtC,MAAMY,QAAQH,WAAWL,GAAG,CAACC;QAC7B,IAAIO,SAASD,UAAUE,KAAK,CAAC,CAACC,MAAQA,MAAMF,MAAMf,MAAM,GAAG;YACzD,4FAA4F;YAC5F,mEAAmE;YACnE,IAAI,CAAC/B,WAAWC,KAAKyB,SAASa,QAAQ;YACtC,KAAK,MAAMS,OAAOH,UAAWD,KAAKJ,IAAI,CAAC;gBAAED;gBAAME,OAAOO;gBAAKC,MAAMH,KAAK,CAACE,IAAI;YAAC;YAC5E;QACF;QACA,4FAA4F;QAC5F,mDAAmD;QACnD,IAAIE;QACJ,IAAI;YACFA,SAAS7C,UAAU;gBAAE8C,SAASZ;gBAAMa,SAASnD,KAAKyB,SAASa;gBAAOc,SAAS;gBAAGC,SAAS;gBAAGC,MAAM;gBAAGC,SAAS,EAAE;gBAAErD,OAAO;YAAK,GAAG;gBAACA;aAAM,EAAEsB,KAAKgC,GAAG,CAACC,SAAS,CAACvD,KAAK;QAClK,EAAE,OAAM;YACN,UAAU,gEAAgE;QAC5E;QACA,KAAK,MAAM6C,OAAOH,UAAW,IAAIK,MAAM,CAACF,IAAI,EAAEJ,KAAKJ,IAAI,CAAC;YAAED;YAAME,OAAOO;YAAKC,MAAMC,MAAM,CAACF,IAAI,CAACC,IAAI;QAAC;IACrG;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAM1C,iBACJiB,OACAG,UACAiB,KAAKe,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;IAGxB,8EAA8E;IAC9E,6DAA6D;IAC7D,MAAMY,SAASzD,SAAS,oBAAoBwC,KAAKb,MAAM;IACvD,IAAK,IAAI+B,IAAI,GAAGA,IAAIlB,KAAKb,MAAM,EAAE+B,KAAKnC,SAASoC,QAAQ,CAAE;QACvD,MAAMC,QAAQpB,KAAKqB,KAAK,CAACH,GAAGA,IAAInC,SAASoC,QAAQ;QACjD,MAAMlC,UAAU,MAAMF,SAASuC,cAAc,CAACF,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEX,IAAI;QACrE,MAAMzB,MAAMK,OAAO,CAACsC,YAAY,CAC9BH,MAAML,GAAG,CAAC,CAACS,KAAKR;YACd,MAAM,EAAE9C,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQmB,OAAO,CAAC+B,EAAE,EAAE5B,WAAW;YACpD,MAAMqC,IAAI,IAAIC,UAAUtC;YACxB,IAAK,IAAIf,IAAI,GAAGA,IAAIe,WAAWf,IAAKoD,CAAC,CAACpD,EAAE,GAAGC,KAAKqD,KAAK,CAACzD,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEmB,MAAM6B,IAAI7B,IAAI;gBAAEE,OAAO2B,IAAI3B,KAAK;gBAAErB;gBAAOoD,QAAQC,OAAOC,IAAI,CAACL,EAAEM,MAAM;YAAE;QAClF;QAEFd,OAAOe,IAAI,CAAC1D,KAAKe,GAAG,CAAC6B,IAAInC,SAASoC,QAAQ,EAAEnB,KAAKb,MAAM;IACzD;IACA8B,OAAOgB,MAAM;AACf;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAeC,mBAAmBtD,KAAY,EAAEC,GAAW,EAAEsD,KAAa,EAAEC,KAAa,EAAEC,OAAqB;QAOvGF;IANd,MAAMrD,UAAU,AAACD,IAAgCC,OAAO;IACxD,IAAI,CAACA,SAAS,MAAM,IAAIxB,WAAW,eAAe;IAClD,MAAMqB,aAAaC,OAAOC,KAAKC;IAE/B,MAAMC,WAAW,MAAMnB,YAAYiB;IACnC,MAAMO,YAAYd,KAAKe,GAAG,CAACxB,YAAYkB,SAASf,IAAI;IACpD,MAAMqC,OAAO,EAAC8B,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAEI,MAAM,CAAC,CAACC,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAInF,IAAI,CAAC;IAClH,MAAM,EAAEa,GAAGwE,EAAE,EAAE,GAAG5E,QAAQ,MAAMiB,SAAS4D,UAAU,CAACtC,OAAOjB,WAAW;IAEtE,OAAOR,MAAMK,OAAO,CAAC2D,UAAU,CAACF,IAAItD,WAAWgD,OAAOC;AACxD;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAeQ,aAAajE,KAAY,EAAEe,IAAY;IAC3D,OAAOf,MAAMK,OAAO,CAAC6D,SAAS,CAACnD;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeoD,aAAanE,KAAY,EAAEoE,IAAY,EAAErD,IAAY,EAAEsD,IAAgE;IAC3I,OAAOrE,MAAMK,OAAO,CAACiE,OAAO,CAACvD,MAAMsD;AACrC"}

package/dist/esm/embed/registry.js

@@ -18,8 +18,7 @@ export function getProvider(cfg) {
     const factory = REGISTRY[e.provider];
     if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider "${e.provider}" has no provider implementation in this build`);
     // Every input that shapes the provider is in the key: two trees sharing an endpoint and
-    // model but declaring different languages or reading different key env vars are two
-    // providers, not one, however long a process lives.
+    // model but declaring different languages or key env vars are two providers, not one.
     const sig = JSON.stringify([
         e.provider,
         e.model,

package/dist/esm/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or reading different key env vars are two\n  // providers, not one, however long a process lives.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","getProvider","cfg","e","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQR,eAAeK,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,OAAO,SAASC,YAAYC,GAAW;QAQYC,QAAaA,QAAaA;IAP3E,MAAMA,IAAIhB,YAAYe;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAC/C,MAAMgB,UAAUZ,QAAQ,CAACW,EAAEE,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAIhB,WAAW,eAAe,CAAC,gBAAgB,EAAEe,EAAEE,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,wFAAwF;IACxF,oFAAoF;IACpF,oDAAoD;IACpD,MAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEE,QAAQ;QAAEF,EAAET,KAAK;SAAES,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,SAAAA,EAAEN,GAAG,cAALM,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIO,IAAIX,UAAUY,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAET,KAAK,EAAES,EAAEP,GAAG,EAAEO,EAAEN,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIM,EAAEM,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,CAACC,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEJ,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzEV,UAAUe,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC,IAAMhB,UAAUiB,MAAM,CAACV;IACjC;IACA,OAAOI;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","getProvider","cfg","e","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQR,eAAeK,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,OAAO,SAASC,YAAYC,GAAW;QAOYC,QAAaA,QAAaA;IAN3E,MAAMA,IAAIhB,YAAYe;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAC/C,MAAMgB,UAAUZ,QAAQ,CAACW,EAAEE,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAIhB,WAAW,eAAe,CAAC,gBAAgB,EAAEe,EAAEE,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,wFAAwF;IACxF,sFAAsF;IACtF,MAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEE,QAAQ;QAAEF,EAAET,KAAK;SAAES,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,SAAAA,EAAEN,GAAG,cAALM,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIO,IAAIX,UAAUY,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAET,KAAK,EAAES,EAAEP,GAAG,EAAEO,EAAEN,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIM,EAAEM,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,CAACC,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEJ,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzEV,UAAUe,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC,IAAMhB,UAAUiB,MAAM,CAACV;IACjC;IACA,OAAOI;AACT"}

package/dist/esm/embed/static.d.ts

@@ -1,2 +1,2 @@
 import type { EmbedProvider } from './types.js';
-export declare function staticProvider(model: string): Promise<EmbedProvider>;
+export declare function staticProvider(model: string, root?: string): Promise<EmbedProvider>;

package/dist/esm/embed/static.js

@@ -4,25 +4,24 @@ import { join } from 'node:path';
 import { SenseError } from '../errors.js';
 import { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.js';
 const BATCH_CAP = 64;
-// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's
-// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.
-// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.
+// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every
+// command's module graph, so it's deferred to a synchronous require inside staticProvider.
 const _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;
 // Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no
 // special tokens, drop unk ids, mean-pool, L2-normalize.
-export async function staticProvider(model) {
+export async function staticProvider(model, root) {
     var _ref;
     var _tokenizerJson_model, _tokenizerJson_model_vocab, _tokenizerJson_model1;
     let dir;
     let languages;
     if (isDownloadable(model)) {
         // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.
-        dir = await downloadModel(model, (file, into)=>console.error(`sense: fetching ${model}/${file} into ${into}`));
+        dir = await downloadModel(model, (file, into)=>console.error(`sense: fetching ${model}/${file} into ${into}`), root);
         // Cached beside the repo; a local model directory has no card to read, so it has none.
-        const cached = readLanguages(model);
+        const cached = readLanguages(model, root);
         languages = cached && cached.length > 0 ? cached : undefined;
     } else {
-        dir = modelDir(model);
+        dir = modelDir(model, root);
         for (const file of MODEL_FILES){
             if (!existsSync(join(dir, file))) {
                 throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);
@@ -43,9 +42,8 @@ export async function staticProvider(model) {
     const tok = new Tokenizer(tokenizerJson, {});
     const unkId = (_ref = (_tokenizerJson_model1 = tokenizerJson.model) === null || _tokenizerJson_model1 === void 0 ? void 0 : (_tokenizerJson_model_vocab = _tokenizerJson_model1.vocab) === null || _tokenizerJson_model_vocab === void 0 ? void 0 : _tokenizerJson_model_vocab[(_tokenizerJson_model = tokenizerJson.model) === null || _tokenizerJson_model === void 0 ? void 0 : _tokenizerJson_model.unk_token]) !== null && _ref !== void 0 ? _ref : -1;
     function one(text) {
-        // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an
-        // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the
-        // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.
+        // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it
+        // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.
         const ids = tok.encode(text, {
             add_special_tokens: false
         }).ids.filter((id)=>Number.isInteger(id) && id !== unkId);

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import put it on every command's\n// module graph regardless of whether the semantic path ran, shipped as the 0.17.0 regression.\n// Deferred to a synchronous require inside staticProvider so only the embed path pays for it.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`));\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for tokens outside the vocab; an\n    // undefined id would index the matrix at NaN and poison the whole mean-pool -- and the\n    // int8 conversion then stores the NaN vector as all zeros, silently. Keep integers only.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,gGAAgG;AAChG,8FAA8F;AAC9F,8FAA8F;AAC9F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa;;QA+BLC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAId,eAAeW,QAAQ;QACzB,qFAAqF;QACrFE,MAAM,MAAMd,cAAcY,OAAO,CAACI,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAEP,MAAM,CAAC,EAAEI,KAAK,MAAM,EAAEC,MAAM;QAC9G,uFAAuF;QACvF,MAAMG,SAASf,cAAcO;QAC7BG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMV,SAASQ;QACf,KAAK,MAAMI,QAAQb,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKgB,KAAKE,QAAQ;gBAChC,MAAM,IAAIjB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEE,IAAI,YAAY,EAAEZ,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMqB,MAAM3B,aAAaE,KAAKgB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAItC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM0B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAACjC,aAAaE,KAAKgB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAGzC,SAAS;IAC/B,MAAM0C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcD,KAAK,cAAnBC,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcD,KAAK,cAAnBC,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,mFAAmF;QACnF,uFAAuF;QACvF,yFAAyF;QACzF,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAE/C,OAAO;QAAE2B;QAAM4B,UAAU7D;QAAWS;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QAAE4B;QAAM4B,UAAU9D;QAAWU;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}

package/dist/esm/embed/store.d.ts

@@ -4,5 +4,5 @@ export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir,
 export declare function modelPresent(cfg: Config): boolean;
 type ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;
 export declare function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig;
-export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string>;
-export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string>;
+export declare function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;
+export declare function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root?: string): Promise<string>;

package/dist/esm/embed/store.js

@@ -4,11 +4,10 @@ import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { writeFileAtomic } from '../lib/atomic-write.js';
 import { fetchWithRetry } from './http.js';
-import { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.js';
+import { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.js';
 import { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.js';
 // identity.ts holds paths and cache identity, no network; store.ts holds the network fetches
-// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise
-// only need store.ts's network surface.
+// (downloads, sha resolution, language metadata), re-exported here for its network surface.
 export { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.js';
 // A non-static provider has nothing to download here; an unreachable endpoint surfaces as an
 // EMBED_MODEL error at call time instead.
@@ -24,8 +23,8 @@ export function localModelMissing(e) {
 }
 // `main` resolves to a sha once per model: a recorded ref pins until its snapshot is
 // removed, so only the first need for an id ever costs this HEAD request.
-async function resolveSha(model) {
-    const existing = readRef(model);
+async function resolveSha(model, root = DEFAULT_MODELS_ROOT) {
+    const existing = readRef(model, root);
     if (existing) return existing;
     const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;
     // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN
@@ -41,13 +40,11 @@ async function resolveSha(model) {
     }
     const sha = res.headers.get('x-repo-commit');
     if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);
-    writeRef(model, sha);
+    writeRef(model, sha, root);
     return sha;
 }
-// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has
-// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an
-// array here). Tags are a fallback for cards that only tag languages, filtered to recognized
-// codes so an unrelated short tag (format markers etc.) is never mistaken for one.
+// cardData.language is the structured field; tags are a fallback for cards that only tag
+// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.
 function languagesFromCard(info) {
     var _info_tags;
     var _info_cardData;
@@ -59,31 +56,39 @@ function languagesFromCard(info) {
     const tagLangs = ((_info_tags = info.tags) !== null && _info_tags !== void 0 ? _info_tags : []).filter(isKnownLanguageTag);
     return tagLangs.length > 0 ? tagLangs : undefined;
 }
-// Best-effort, cached forever once resolved (even to "none"): unlike model weights, missing or
-// unreachable metadata is not an error -- it just means the language-fit check stays off.
-async function ensureLanguages(model) {
+// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:
+// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).
+function cardUnreadable(model) {
+    console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \`sense download\`)`);
+}
+async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     var _languagesFromCard;
-    if (existsSync(languagesPath(model))) return;
+    if (existsSync(languagesPath(model, root))) return;
     // A measured entry needs no card and no network, and stays deterministic offline.
     const measured = MEASURED_MODEL_LANGUAGES[model];
     if (measured) {
-        writeLanguages(model, measured);
+        writeLanguages(model, measured, root);
         return;
     }
     let res;
     try {
         res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});
     } catch  {
+        cardUnreadable(model);
+        return;
+    }
+    if (!res.ok) {
+        cardUnreadable(model);
         return;
     }
-    if (!res.ok) return;
     let info;
     try {
         info = await res.json();
     } catch  {
+        cardUnreadable(model);
         return;
     }
-    writeLanguages(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : []);
+    writeLanguages(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : [], root);
 }
 async function fetchToFile(url, dest) {
     let res;
@@ -97,8 +102,8 @@ async function fetchToFile(url, dest) {
 }
 // Fetches an already-known sha straight into its snapshot dir, without touching refs/main --
 // for downloadModel below, and for callers pinning an exact revision (the parity suite).
-export async function downloadModelRevision(model, sha, onFile) {
-    const dir = snapshotDir(model, sha);
+export async function downloadModelRevision(model, sha, onFile, root = DEFAULT_MODELS_ROOT) {
+    const dir = snapshotDir(model, sha, root);
     mkdirSync(dir, {
         recursive: true
     });
@@ -111,13 +116,13 @@ export async function downloadModelRevision(model, sha, onFile) {
 }
 // The only code path that resolves `main`: a static provider's first construction calls it
 // lazily, `sense download` as an explicit prefetch. Idempotent either way.
-export async function downloadModel(model, onFile) {
+export async function downloadModel(model, onFile, root = DEFAULT_MODELS_ROOT) {
     if (!isDownloadable(model)) {
         throw new SenseError('EMBED_MODEL', `embed.model "${model}" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like "minishlab/potion-retrieval-32M"`);
     }
-    const sha = await resolveSha(model);
+    const sha = await resolveSha(model, root);
     // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first
     // construction and an explicit prefetch both end up with a cached language verdict.
-    await ensureLanguages(model);
-    return downloadModelRevision(model, sha, onFile);
+    await ensureLanguages(model, root);
+    return downloadModelRevision(model, sha, onFile, root);
 }

package/dist/esm/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata). Re-exported here for callers that otherwise\n// only need store.ts's network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string): Promise<string> {\n  const existing = readRef(model);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field (verified live: minishlab/potion-retrieval-32M has\n// neither this nor any language tag, so it stays silent; potion-multilingual-128M carries an\n// array here). Tags are a fallback for cards that only tag languages, filtered to recognized\n// codes so an unrelated short tag (format markers etc.) is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Best-effort, cached forever once resolved (even to \"none\"): unlike model weights, missing or\n// unreachable metadata is not an error -- it just means the language-fit check stays off.\nasync function ensureLanguages(model: string): Promise<void> {\n  if (existsSync(languagesPath(model))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    return;\n  }\n  if (!res.ok) return;\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? []);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  const dir = snapshotDir(model, sha);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model);\n  return downloadModelRevision(model, sha, onFile);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","existing","url","res","method","redirect","err","message","sha","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","undefined","ensureLanguages","measured","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAASC,cAAc,QAAQ,YAAY;AAC3C,SAASC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC1I,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,8FAA8F;AAC9F,wCAAwC;AACxC,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIpB,YAAYmB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa;IACrC,MAAMG,WAAWjB,QAAQc;IACzB,IAAIG,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEJ,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,IAAIK;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAeuB,KAAK;YAAEE,QAAQ;YAAQC,UAAU;QAAS;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI7B,WAAW,eAAe,CAAC,gBAAgB,EAAEyB,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,MAAMC,MAAML,IAAIM,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAI/B,WAAW,eAAe,GAAGyB,IAAI,0CAA0C,CAAC;IAChGf,SAASW,OAAOU;IAChB,OAAOA;AACT;AAOA,+FAA+F;AAC/F,6FAA6F;AAC7F,6FAA6F;AAC7F,mFAAmF;AACnF,SAASG,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,MAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACjC;IAC1C,OAAO+B,SAASD,MAAM,GAAG,IAAIC,WAAWG;AAC1C;AAEA,+FAA+F;AAC/F,0FAA0F;AAC1F,eAAeC,gBAAgBzB,KAAa;QAqBpBa;IApBtB,IAAItC,WAAWS,cAAcgB,SAAS;IACtC,kFAAkF;IAClF,MAAM0B,WAAWnC,wBAAwB,CAACS,MAAM;IAChD,IAAI0B,UAAU;QACZtC,eAAeY,OAAO0B;QACtB;IACF;IACA,IAAIrB;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAe,CAAC,kCAAkC,EAAEmB,OAAO,EAAE,CAAC;IAC5E,EAAE,OAAM;QACN;IACF;IACA,IAAI,CAACK,IAAIsB,EAAE,EAAE;IACb,IAAIb;IACJ,IAAI;QACFA,OAAQ,MAAMT,IAAIuB,IAAI;IACxB,EAAE,OAAM;QACN;IACF;IACAxC,eAAeY,QAAOa,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE;AACrD;AAEA,eAAegB,YAAYzB,GAAW,EAAE0B,IAAY;IAClD,IAAIzB;IACJ,IAAI;QACFA,MAAM,MAAMxB,eAAeuB,KAAK,CAAC;IACnC,EAAE,OAAOI,KAAK;QACZ,MAAM,IAAI7B,WAAW,eAAe,CAAC,gBAAgB,EAAEyB,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,IAAI,CAACJ,IAAIsB,EAAE,EAAE,MAAM,IAAIhD,WAAW,eAAe,GAAGyB,IAAI,SAAS,EAAEC,IAAI0B,MAAM,EAAE;IAC/EnD,gBAAgBkD,MAAME,OAAOC,IAAI,CAAC,MAAM5B,IAAI6B,WAAW;AACzD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeC,sBAAsBnC,KAAa,EAAEU,GAAW,EAAE0B,MAA4C;IAClH,MAAMC,MAAMlD,YAAYa,OAAOU;IAC/BlC,UAAU6D,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQtD,YAAa;QAC9B,IAAIV,WAAWE,KAAK4D,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMR,YAAY,CAAC,uBAAuB,EAAE7B,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE6B,MAAM,EAAE9D,KAAK4D,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcxC,KAAa,EAAEoC,MAA4C;IAC7F,IAAI,CAACrD,eAAeiB,QAAQ;QAC1B,MAAM,IAAIrB,WAAW,eAAe,CAAC,aAAa,EAAEqB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF;IAC7B,sFAAsF;IACtF,oFAAoF;IACpF,MAAMyB,gBAAgBzB;IACtB,OAAOmC,sBAAsBnC,OAAOU,KAAK0B;AAC3C"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","err","message","sha","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","undefined","cardUnreadable","console","error","ensureLanguages","measured","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAASC,cAAc,QAAQ,YAAY;AAC3C,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIrB,YAAYoB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,IAAIM;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK;YAAEE,QAAQ;YAAQC,UAAU;QAAS;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,MAAMC,MAAML,IAAIM,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIjC,WAAW,eAAe,GAAG2B,IAAI,0CAA0C,CAAC;IAChGhB,SAASW,OAAOW,KAAKR;IACrB,OAAOQ;AACT;AAOA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASG,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,MAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAAClC;IAC1C,OAAOgC,SAASD,MAAM,GAAG,IAAIC,WAAWG;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASC,eAAe1B,KAAa;IACnC2B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAE5B,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAe6B,gBAAgB7B,KAAa,EAAEG,OAAetB,mBAAmB;QA0BxDiC;IAzBtB,IAAIxC,WAAWU,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM2B,WAAWvC,wBAAwB,CAACS,MAAM;IAChD,IAAI8B,UAAU;QACZ1C,eAAeY,OAAO8B,UAAU3B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAe,CAAC,kCAAkC,EAAEoB,OAAO,EAAE,CAAC;IAC5E,EAAE,OAAM;QACN0B,eAAe1B;QACf;IACF;IACA,IAAI,CAACM,IAAIyB,EAAE,EAAE;QACXL,eAAe1B;QACf;IACF;IACA,IAAIe;IACJ,IAAI;QACFA,OAAQ,MAAMT,IAAI0B,IAAI;IACxB,EAAE,OAAM;QACNN,eAAe1B;QACf;IACF;IACAZ,eAAeY,QAAOc,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAEX;AACvD;AAEA,eAAe8B,YAAY5B,GAAW,EAAE6B,IAAY;IAClD,IAAI5B;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK,CAAC;IACnC,EAAE,OAAOI,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,IAAI,CAACJ,IAAIyB,EAAE,EAAE,MAAM,IAAIrD,WAAW,eAAe,GAAG2B,IAAI,SAAS,EAAEC,IAAI6B,MAAM,EAAE;IAC/ExD,gBAAgBuD,MAAME,OAAOC,IAAI,CAAC,MAAM/B,IAAIgC,WAAW;AACzD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeC,sBAAsBvC,KAAa,EAAEW,GAAW,EAAE6B,MAA4C,EAAErC,OAAetB,mBAAmB;IACtJ,MAAM4D,MAAMtD,YAAYa,OAAOW,KAAKR;IACpC5B,UAAUkE,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQ1D,YAAa;QAC9B,IAAIX,WAAWE,KAAKiE,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMR,YAAY,CAAC,uBAAuB,EAAEjC,MAAM,SAAS,EAAEW,IAAI,CAAC,EAAEgC,MAAM,EAAEnE,KAAKiE,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAc5C,KAAa,EAAEwC,MAA4C,EAAErC,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAItB,WAAW,eAAe,CAAC,aAAa,EAAEsB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMW,MAAM,MAAMT,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM0B,gBAAgB7B,OAAOG;IAC7B,OAAOoC,sBAAsBvC,OAAOW,KAAK6B,QAAQrC;AACnD"}

package/dist/esm/embed/types.js

@@ -1,5 +1,3 @@
 // Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):
-// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]
-// column width, so both stores share one source of truth for it rather than a schema
-// constant drifting from the quantization one.
+// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.
 export const STORE_DIMS = 256;

package/dist/esm/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused. Also the duckdb store's FLOAT[N]\n// column width, so both stores share one source of truth for it rather than a schema\n// constant drifting from the quantization one.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,2FAA2F;AAC3F,qFAAqF;AACrF,+CAA+C;AAC/C,OAAO,MAAMA,aAAa,IAAI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}

package/dist/esm/errors.d.ts

@@ -1,4 +1,4 @@
-export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING';
+export type SenseErrorCode = 'CONFIG_NOT_FOUND' | 'CONFIG_EXISTS' | 'CONFIG_INVALID' | 'CONFIG_VERSION_UNSUPPORTED' | 'WATCH_ACTIVE' | 'NOTE_NOT_FOUND' | 'NOTE_AMBIGUOUS' | 'EMBED_DISABLED' | 'EMBED_MODEL' | 'EMBED_MODEL_MISSING' | 'EMBED_MODEL_MISMATCH' | 'SEARCH_SYNTAX' | 'COLUMN_LIMIT' | 'PRESET_UNKNOWN' | 'PRESET_NOT_SEMANTIC' | 'NOTE_NOT_EMBEDDED' | 'STORE_UNKNOWN' | 'STORE_DEPENDENCY_MISSING' | 'STORE_CAPABILITY_MISSING' | 'LEXICAL_NULL_PATH';
 export declare class SenseError extends Error {
     code: SenseErrorCode;
     constructor(code: SenseErrorCode, message: string);

package/dist/esm/errors.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAsBrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/errors.ts"],"sourcesContent":["// Library code throws; only cli.ts prints and exits.\nexport type SenseErrorCode =\n  | 'CONFIG_NOT_FOUND'\n  | 'CONFIG_EXISTS'\n  | 'CONFIG_INVALID'\n  | 'CONFIG_VERSION_UNSUPPORTED'\n  | 'WATCH_ACTIVE'\n  | 'NOTE_NOT_FOUND'\n  | 'NOTE_AMBIGUOUS'\n  | 'EMBED_DISABLED'\n  | 'EMBED_MODEL'\n  | 'EMBED_MODEL_MISSING'\n  | 'EMBED_MODEL_MISMATCH'\n  | 'SEARCH_SYNTAX'\n  | 'COLUMN_LIMIT'\n  | 'PRESET_UNKNOWN'\n  | 'PRESET_NOT_SEMANTIC'\n  | 'NOTE_NOT_EMBEDDED'\n  | 'STORE_UNKNOWN'\n  | 'STORE_DEPENDENCY_MISSING'\n  | 'STORE_CAPABILITY_MISSING'\n  | 'LEXICAL_NULL_PATH';\n\nexport class SenseError extends Error {\n  code: SenseErrorCode;\n\n  constructor(code: SenseErrorCode, message: string) {\n    super(message);\n    this.name = 'SenseError';\n    this.code = code;\n  }\n}\n"],"names":["SenseError","Error","code","message","name"],"mappings":"AAAA,qDAAqD;AAuBrD,OAAO,MAAMA,mBAAmBC;IAG9B,YAAYC,IAAoB,EAAEC,OAAe,CAAE;QACjD,KAAK,CAACA;QACN,IAAI,CAACC,IAAI,GAAG;QACZ,IAAI,CAACF,IAAI,GAAGA;IACd;AACF"}

package/dist/esm/features/embed.js

@@ -1,22 +1,17 @@
 import { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.js';
 import { embedConfig } from '../config/index.js';
+import { stashChunkText } from '../embed/handoff.js';
 import { modelIdentity } from '../embed/identity.js';
-// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary
-// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.
-//
-// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw
-// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw
-// put the frontmatter block in its own leading chunk on every note with a heading, which made
-// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also
-// disagreed with FTS, which indexes the body alone.
+import { countLines } from '../scan/frontmatter.js';
+// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
+// `offset` shifts line numbers back onto raw and a range stays a direct Read range.
 function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
     const prefix = [
         search === null || search === void 0 ? void 0 : search.title,
         search === null || search === void 0 ? void 0 : search.summary
     ].filter(Boolean).join('\n');
-    // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change
-    // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker
-    // input not already carried in config.
+    // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature
+    // for every chunker input not already carried in config.
     return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? {
         targetTokens: chunkTokens
     } : undefined).map((c)=>({
@@ -34,7 +29,7 @@ export const embed = {
         var _cfg_embed;
         // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.
         // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
-        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : parse(body), body, search, raw.split('\n').length - body.split('\n').length, cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
+        return chunksOf(blocks !== null && blocks !== void 0 ? blocks : parse(body), body, search, countLines(raw) - countLines(body), cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
     async remove (db, paths) {
         if (paths.length === 0) return;
@@ -46,24 +41,29 @@ export const embed = {
     // filter skips it), so extracted is undefined here -- those docs contribute no rows.
     async store (db, docs) {
         const rows = [];
+        const texts = new Map();
         for (const { path, extracted } of docs){
             if (!extracted) continue;
-            extracted.forEach((c, idx)=>rows.push([
+            const chunks = extracted;
+            chunks.forEach((c, idx)=>rows.push([
                     path,
                     idx,
                     c.startLine,
                     c.endLine
                 ]));
+            texts.set(path, chunks.map((c)=>c.text));
         }
         if (rows.length === 0) return;
         await db.runBatch('INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);
+        // The text extract() just computed, handed to the embed pass that follows in this same
+        // process rather than re-derived from the file there (src/embed/handoff.ts).
+        stashChunkText(db, texts);
     },
     enabledForFile (_cfg, file) {
         return file.embed;
     },
-    // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it "embed
-    // settings". Driven by embedConfig(cfg) directly (a model can be configured with no preset
-    // yet using vectors), not by whether this feature is itself active.
+    // Keyed so a rebuild notice can name it "embed settings". Driven by embedConfig(cfg) directly:
+    // a model can be configured with no preset yet using vectors.
     signature (cfg) {
         const e = embedConfig(cfg);
         if (!e) return 'embed:off';

package/dist/esm/features/embed.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded. Reconcile writes dirty\n// rows; embedding tops up on the next search, so staleness costs recall. Providers, the vector\n// fill, and the search-time helpers live under src/embed/. Block/group structure (D1-D5) lives\n// in src/chunk; this keeps only the title/summary prefix and the empty-chunk drop (D6).\n\nexport type { Chunk };\n\n// Deterministic, so embed time can re-derive text from stored line ranges; the title/summary\n// prefix mirrors bm25 field weighting, and an empty chunk is dropped before the prefix is added.\n//\n// Chunks the body, not the raw file, with `offset` shifting line numbers back onto the raw\n// file so a range stays a direct Read range (sections is 1-indexed over raw too). Chunking raw\n// put the frontmatter block in its own leading chunk on every note with a heading, which made\n// `lines` point at YAML and made frontmatter-only notes near-identical to each other. It also\n// disagreed with FTS, which indexes the body alone.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // chunk()'s default text mode is 'raw' (D9), the shipped choice. A hardcoded option change\n  // here must bump src/chunk/version.ts's CHUNK_VERSION -- the signature for every chunker\n  // input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, raw.split('\\n').length - body.split('\\n').length, cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      (extracted as Chunk[]).forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n    }\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Provider/model/chunk-version/identity, keyed so a rebuild notice can name it \"embed\n  // settings\". Driven by embedConfig(cfg) directly (a model can be configured with no preset\n  // yet using vectors), not by whether this feature is itself active.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","modelIdentity","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","split","length","remove","paths","runBatch","p","store","docs","rows","path","extracted","forEach","idx","push","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,aAAa,QAAQ,uBAAuB;AAUrD,6FAA6F;AAC7F,iGAAiG;AACjG,EAAE;AACF,2FAA2F;AAC3F,+FAA+F;AAC/F,8FAA8F;AAC9F,8FAA8F;AAC9F,oDAAoD;AACpD,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,2FAA2F;IAC3F,yFAAyF;IACzF,uCAAuC;IACvC,OAAOf,gBAAgBK,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAChL;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG4D0B;QAFvG,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUJ,MAAMK,OAAOA,MAAMC,QAAQsB,IAAIG,KAAK,CAAC,MAAMC,MAAM,GAAG3B,KAAK0B,KAAK,CAAC,MAAMC,MAAM,EAAEF,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAChI;IACA,MAAMyB,QAAOR,EAAE,EAAES,KAAK;QACpB,IAAIA,MAAMF,MAAM,KAAK,GAAG;QACxB,MAAMP,GAAGU,QAAQ,CACf,2CACAD,MAAMjB,GAAG,CAAC,CAACmB,IAAM;gBAACA;aAAE;IAExB;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMC,OAAMZ,EAAE,EAAEa,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIH,KAAM;YACtC,IAAI,CAACG,WAAW;YACfA,UAAsBC,OAAO,CAAC,CAACxB,GAAGyB,MAAQJ,KAAKK,IAAI,CAAC;oBAACJ;oBAAMG;oBAAKzB,EAAEC,SAAS;oBAAED,EAAEE,OAAO;iBAAC;QAC1F;QACA,IAAImB,KAAKP,MAAM,KAAK,GAAG;QACvB,MAAMP,GAAGU,QAAQ,CAAC,+GAA+GI;IACnI;IACAM,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAKzB,KAAK;IACnB;IACA,sFAAsF;IACtF,2FAA2F;IAC3F,oEAAoE;IACpE0B,WAAUlB,GAAG;QACX,MAAMmB,IAAIhD,YAAY6B;QACtB,IAAI,CAACmB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAWjD,cAAc+C,EAAEG,KAAK,IAAIrC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMsC,eAAeJ,EAAEzC,WAAW,KAAKO,YAAY,GAAGjB,cAAc,CAAC,EAAEmD,EAAEzC,WAAW,EAAE,GAAGV;QACzF,OAAO,CAAC,MAAM,EAAEmD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAanC,YAAY,CAAC,CAAC,EAAEmC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/features/embed.ts"],"sourcesContent":["import type { Block, Chunk } from '../chunk/index.ts';\nimport { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { stashChunkText } from '../embed/handoff.ts';\nimport { modelIdentity } from '../embed/identity.ts';\nimport { countLines } from '../scan/frontmatter.ts';\nimport type { Feature } from './types.ts';\n\n// int8 vectors with a per-vector scale, NULL vector = not yet embedded: reconcile writes dirty\n// rows and embedding tops up on the next search, so staleness costs recall.\n\nexport type { Chunk };\n\n// The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so\n// `offset` shifts line numbers back onto raw and a range stays a direct Read range.\nfunction chunksOf(blocks: Block[], body: string, search?: { title: string; summary: string }, offset = 0, chunkTokens?: number): Chunk[] {\n  const prefix = [search?.title, search?.summary].filter(Boolean).join('\\n');\n  // A hardcoded option change here must bump src/chunk/version.ts's CHUNK_VERSION, the signature\n  // for every chunker input not already carried in config.\n  return chunkFromBlocks(blocks, body, chunkTokens !== undefined ? { targetTokens: chunkTokens } : undefined).map((c) => ({\n    startLine: c.startLine + offset,\n    endLine: c.endLine + offset,\n    text: prefix ? `${prefix}\\n${c.text}` : c.text,\n  }));\n}\n\nexport const embed: Feature = {\n  name: 'embed',\n  async schema(db) {\n    await db.exec(`CREATE TABLE IF NOT EXISTS embeddings (\"path\" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY (\"path\", chunk))`);\n  },\n  extract(raw, body, search, _data, cfg, blocks) {\n    // Lines the frontmatter occupies, so body line 1 maps back to its raw line number.\n    // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.\n    return chunksOf(blocks ?? parse(body), body, search, countLines(raw) - countLines(body), cfg?.embed?.chunkTokens);\n  },\n  async remove(db, paths) {\n    if (paths.length === 0) return;\n    await db.runBatch(\n      'DELETE FROM embeddings WHERE \"path\" = ?',\n      paths.map((p) => [p])\n    );\n  },\n  // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file\n  // filter skips it), so extracted is undefined here -- those docs contribute no rows.\n  async store(db, docs) {\n    const rows: unknown[][] = [];\n    const texts = new Map<string, string[]>();\n    for (const { path, extracted } of docs) {\n      if (!extracted) continue;\n      const chunks = extracted as Chunk[];\n      chunks.forEach((c, idx) => rows.push([path, idx, c.startLine, c.endLine]));\n      texts.set(\n        path,\n        chunks.map((c) => c.text)\n      );\n    }\n    if (rows.length === 0) return;\n    await db.runBatch('INSERT INTO embeddings (\"path\", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);\n    // The text extract() just computed, handed to the embed pass that follows in this same\n    // process rather than re-derived from the file there (src/embed/handoff.ts).\n    stashChunkText(db, texts);\n  },\n  enabledForFile(_cfg, file) {\n    return file.embed;\n  },\n  // Keyed so a rebuild notice can name it \"embed settings\". Driven by embedConfig(cfg) directly:\n  // a model can be configured with no preset yet using vectors.\n  signature(cfg) {\n    const e = embedConfig(cfg);\n    if (!e) return 'embed:off';\n    // Weight identity from identity.ts, no network: a static model's resolved sha or local\n    // size+mtime, appended once known so changed weights re-embed.\n    const identity = e.provider === 'static' ? modelIdentity(e.model) : undefined;\n    // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker\n    // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.\n    const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;\n    return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;\n  },\n};\n"],"names":["CHUNK_VERSION","chunkFromBlocks","parse","embedConfig","stashChunkText","modelIdentity","countLines","chunksOf","blocks","body","search","offset","chunkTokens","prefix","title","summary","filter","Boolean","join","undefined","targetTokens","map","c","startLine","endLine","text","embed","name","schema","db","exec","extract","raw","_data","cfg","remove","paths","length","runBatch","p","store","docs","rows","texts","Map","path","extracted","chunks","forEach","idx","push","set","enabledForFile","_cfg","file","signature","e","identity","provider","model","chunkVersion"],"mappings":"AACA,SAASA,aAAa,EAAEC,eAAe,EAAEC,KAAK,QAAQ,oBAAoB;AAC1E,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,cAAc,QAAQ,sBAAsB;AACrD,SAASC,aAAa,QAAQ,uBAAuB;AACrD,SAASC,UAAU,QAAQ,yBAAyB;AAQpD,+FAA+F;AAC/F,oFAAoF;AACpF,SAASC,SAASC,MAAe,EAAEC,IAAY,EAAEC,MAA2C,EAAEC,SAAS,CAAC,EAAEC,WAAoB;IAC5H,MAAMC,SAAS;QAACH,mBAAAA,6BAAAA,OAAQI,KAAK;QAAEJ,mBAAAA,6BAAAA,OAAQK,OAAO;KAAC,CAACC,MAAM,CAACC,SAASC,IAAI,CAAC;IACrE,+FAA+F;IAC/F,yDAAyD;IACzD,OAAOjB,gBAAgBO,QAAQC,MAAMG,gBAAgBO,YAAY;QAAEC,cAAcR;IAAY,IAAIO,WAAWE,GAAG,CAAC,CAACC,IAAO,CAAA;YACtHC,WAAWD,EAAEC,SAAS,GAAGZ;YACzBa,SAASF,EAAEE,OAAO,GAAGb;YACrBc,MAAMZ,SAAS,GAAGA,OAAO,EAAE,EAAES,EAAEG,IAAI,EAAE,GAAGH,EAAEG,IAAI;QAChD,CAAA;AACF;AAEA,OAAO,MAAMC,QAAiB;IAC5BC,MAAM;IACN,MAAMC,QAAOC,EAAE;QACb,MAAMA,GAAGC,IAAI,CAAC,CAAC,8JAA8J,CAAC;IAChL;IACAC,SAAQC,GAAG,EAAEvB,IAAI,EAAEC,MAAM,EAAEuB,KAAK,EAAEC,GAAG,EAAE1B,MAAM;YAG8C0B;QAFzF,mFAAmF;QACnF,4FAA4F;QAC5F,OAAO3B,SAASC,mBAAAA,oBAAAA,SAAUN,MAAMO,OAAOA,MAAMC,QAAQJ,WAAW0B,OAAO1B,WAAWG,OAAOyB,gBAAAA,2BAAAA,aAAAA,IAAKR,KAAK,cAAVQ,iCAAAA,WAAYtB,WAAW;IAClH;IACA,MAAMuB,QAAON,EAAE,EAAEO,KAAK;QACpB,IAAIA,MAAMC,MAAM,KAAK,GAAG;QACxB,MAAMR,GAAGS,QAAQ,CACf,2CACAF,MAAMf,GAAG,CAAC,CAACkB,IAAM;gBAACA;aAAE;IAExB;IACA,8FAA8F;IAC9F,qFAAqF;IACrF,MAAMC,OAAMX,EAAE,EAAEY,IAAI;QAClB,MAAMC,OAAoB,EAAE;QAC5B,MAAMC,QAAQ,IAAIC;QAClB,KAAK,MAAM,EAAEC,IAAI,EAAEC,SAAS,EAAE,IAAIL,KAAM;YACtC,IAAI,CAACK,WAAW;YAChB,MAAMC,SAASD;YACfC,OAAOC,OAAO,CAAC,CAAC1B,GAAG2B,MAAQP,KAAKQ,IAAI,CAAC;oBAACL;oBAAMI;oBAAK3B,EAAEC,SAAS;oBAAED,EAAEE,OAAO;iBAAC;YACxEmB,MAAMQ,GAAG,CACPN,MACAE,OAAO1B,GAAG,CAAC,CAACC,IAAMA,EAAEG,IAAI;QAE5B;QACA,IAAIiB,KAAKL,MAAM,KAAK,GAAG;QACvB,MAAMR,GAAGS,QAAQ,CAAC,+GAA+GI;QACjI,uFAAuF;QACvF,6EAA6E;QAC7EtC,eAAeyB,IAAIc;IACrB;IACAS,gBAAeC,IAAI,EAAEC,IAAI;QACvB,OAAOA,KAAK5B,KAAK;IACnB;IACA,+FAA+F;IAC/F,8DAA8D;IAC9D6B,WAAUrB,GAAG;QACX,MAAMsB,IAAIrD,YAAY+B;QACtB,IAAI,CAACsB,GAAG,OAAO;QACf,uFAAuF;QACvF,+DAA+D;QAC/D,MAAMC,WAAWD,EAAEE,QAAQ,KAAK,WAAWrD,cAAcmD,EAAEG,KAAK,IAAIxC;QACpE,uFAAuF;QACvF,4FAA4F;QAC5F,MAAMyC,eAAeJ,EAAE5C,WAAW,KAAKO,YAAY,GAAGnB,cAAc,CAAC,EAAEwD,EAAE5C,WAAW,EAAE,GAAGZ;QACzF,OAAO,CAAC,MAAM,EAAEwD,EAAEE,QAAQ,CAAC,CAAC,EAAEF,EAAEG,KAAK,CAAC,CAAC,EAAEC,eAAeH,aAAatC,YAAY,CAAC,CAAC,EAAEsC,UAAU,GAAG,IAAI;IACxG;AACF,EAAE"}

package/dist/esm/features/fences.js

@@ -1,8 +1,5 @@
-// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column
-// 0 -- no indent allowance, a deliberate divergence from CommonMark's up-to-3-space rule (see
-// test/unit/features/fences.test.ts DIVERGENCES table). A backtick opener's info string must not itself
-// contain a backtick (spec rule). A closer is a run of the SAME character, length >= the
-// opener's, holding nothing but trailing spaces after the run.
+// Shared line-fence tracker for tags/sections/embed. Opener: >=3 backticks or tildes at column 0, no indent
+// allowance (diverges from CommonMark's up-to-3-space rule). Closer: same char, length >= opener's, trailing spaces only.
 const BACKTICK_OPEN = /^(`{3,})(.*)$/;
 const TILDE_OPEN = /^(~{3,})(.*)$/;
 export function fenceTracker() {
@@ -80,10 +77,8 @@ export function commentTracker() {
         }
     };
 }
-// CommonMark's HTML-block type-6 list (fixed by the spec, not a drifting enumeration): a line
-// starting with an open or close tag of one of these, at column 0 (leading whitespace/`>`
-// blockquote markers tolerated), opens a block that swallows following lines -- tags, links,
-// and comments alike -- until a blank line closes it.
+// CommonMark's HTML-block type-6 tag list (fixed by the spec). A line opening or closing one of these at
+// column 0 (leading whitespace/`>` tolerated) swallows following lines -- tags, links, comments alike -- until a blank line.
 const HTML_BLOCK_TAGS = new Set([
     'address',
     'article',
@@ -160,12 +155,8 @@ const HTML_PRE_CLOSE_RE = /<\/(?:script|pre|style|textarea)>/i; // any of the fo
 // An opening or closing tag at column 0, tag name captured for the lookups above.
 const HTML_BLOCK_OPEN_RE = /^<\/?([a-zA-Z][a-zA-Z0-9]*)(?:[ \t]|\/?>|$)/;
 const BLANK_LINE_RE = /^[ \t>]*$/; // whitespace/blockquote-markers only -- a paragraph break, and what ends a type-6 block
-// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines
-// preserved, so line numbers and offsets survive) plus inline code spans masked. The single
-// region masker for tags/links: fence and HTML-block delimiter+content lines are opaque, an
-// unclosed comment dies at its containing block's end (or the next blank line at paragraph
-// level -- never silently to end of file), and a closed comment masks exactly its span.
-// Bodies without a backtick/tilde/`<` fast-path out untouched.
+// Body with fenced-code, HTML-block, and <!-- --> comment regions replaced by spaces (newlines preserved, so line numbers and offsets survive), inline code spans masked too.
+// An unclosed comment dies at its block's end or the next paragraph blank line, never silently to end of file; bodies without a backtick/tilde/`<` fast-path out untouched.
 export function maskRegions(body) {
     if (!/[`~<]/.test(body)) return body;
     const fence = fenceTracker();
@@ -195,7 +186,7 @@ export function maskRegions(body) {
         }
         const uncommented = comment.inComment || line.includes('<!--') ? comment.mask(line) : line;
         if (comment.inComment && BLANK_LINE_RE.test(line)) comment.close(); // paragraph-level: dies at the next blank line
-        // Indented or blockquoted HTML blocks still swallow their content in Obsidian, so the
+        // Indented or blockquoted HTML blocks swallow their content in Obsidian too, so the
         // opener test runs after stripping leading whitespace and > markers.
         const stripped = uncommented.replace(/^[ \t>]*/, '');
         if (stripped[0] === '<') {
@@ -222,9 +213,8 @@ export function maskRegions(body) {
         return uncommented.includes('`') ? maskCodeSpans(uncommented) : uncommented;
     }).join('\n');
 }
-// A code span opens on a run of N backticks and closes at the next run of exactly N -- a
-// shorter or longer run in between is literal text, not a delimiter (CommonMark code spans).
-// Masked with spaces so column positions and tag-boundary whitespace are unaffected.
+// A code span opens on a run of N backticks and closes at the next run of exactly N (CommonMark code spans);
+// a shorter/longer run in between is literal text. Masked with spaces so column positions stay unaffected.
 export function maskCodeSpans(line) {
     let out = '';
     let i = 0;