sensemaking 0.25.1 → 0.26.0

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = (await store.vectors.pending()).filter((row) => allowed?.has(row.path) ?? true);\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map<string, string[]>();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && chunkIdxs.every((idx) => idx < texts.length)) {\n      for (const idx of chunkIdxs) jobs.push({ path, chunk: idx, text: texts[idx] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const idx of chunkIdxs) {\n      const chunk = chunks[idx];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      jobs.push({ path, chunk: idx, text: chunk.text });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["SenseError","embed","progress","parseSource","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","store","cfg","allowed","provider","dirty","vectors","pending","filter","row","has","path","length","storeDims","min","byPath","Map","list","get","push","chunk","set","textByPath","sourceStmt","prepare","jobs","chunkIdxs","texts","every","idx","text","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","presets","doc","extracted","map","j","report","i","batchCap","batch","slice","embedDocuments","writeVectors","job","q","Int8Array","round","vector","from","buffer","tick","finish","ensureDocumentEmbeddings","_cfg","semanticCandidates","terms","fetch","rootDir","baseDir","match","t","includes","join","qv","embedQuery","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAE/C,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAASC,UAAU,QAAQ,aAAa;AAExC,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,OAAO,eAAeE,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B;QAatEpB;IAZnB,MAAMqB,WAAW,MAAMnB,YAAYiB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQ,AAAC,CAAA,MAAMJ,MAAMK,OAAO,CAACC,OAAO,EAAC,EAAGC,MAAM,CAAC,CAACC;;uBAAQN,oBAAAA,8BAAAA,QAASO,GAAG,CAACD,IAAIE,IAAI,wCAAK;;IACxF,IAAIN,MAAMO,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYlB,KAAKmB,GAAG,CAAC5B,YAAYkB,SAASf,IAAI;IAEpD,MAAM0B,SAAS,IAAIC;IACnB,KAAK,MAAMP,OAAOJ,MAAO;YACVU;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACT,IAAIE,IAAI,eAAnBI,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACV,IAAIW,KAAK;QACnBL,OAAOM,GAAG,CAACZ,IAAIE,IAAI,EAAEM;IACvB;IAEA,MAAMK,cAAavC,iBAAAA,cAAckB,oBAAdlB,4BAAAA,iBAAwB,IAAIiC;IAC/C,MAAMO,aAAa,MAAMtB,MAAMuB,OAAO,CAAC;IAEvC,MAAMC,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACd,MAAMe,UAAU,IAAIX,OAAQ;QACtC,MAAMY,QAAQL,WAAWJ,GAAG,CAACP;QAC7B,IAAIgB,SAASD,UAAUE,KAAK,CAAC,CAACC,MAAQA,MAAMF,MAAMf,MAAM,GAAG;YACzD,KAAK,MAAMiB,OAAOH,UAAWD,KAAKN,IAAI,CAAC;gBAAER;gBAAMS,OAAOS;gBAAKC,MAAMH,KAAK,CAACE,IAAI;YAAC;YAC5E;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAME,SAAU,MAAMR,WAAWL,GAAG,CAACP;QACrC,IAAI,CAACoB,QAAQ,MAAM,IAAIpD,WAAW,mBAAmB,CAAC,mCAAmC,EAAEgC,KAAK,kGAAkG,CAAC;QACnM,MAAMqB,SAASlD,YAAY;YAAEmD,SAAStB;YAAMuB,SAASvB;YAAMwB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOD,IAAI;YAAGU,SAAS,EAAE;YAAE5D,OAAO;QAAK,GAAGmD,OAAOD,IAAI,EAAE;YAAClD;SAAM,EAAEsB,KAAKuC,GAAG,CAACC,SAAS,CAAC9D,KAAK;QACnM,KAAK,MAAMiD,OAAOH,UAAW;YAC3B,MAAMN,QAAQY,MAAM,CAACH,IAAI;YACzB,IAAI,CAACT,OAAO,MAAM,IAAIzC,WAAW,mBAAmB,CAAC,4BAA4B,EAAEgC,KAAK,2GAA2G,CAAC;YACpMc,KAAKN,IAAI,CAAC;gBAAER;gBAAMS,OAAOS;gBAAKC,MAAMV,MAAMU,IAAI;YAAC;QACjD;IACF;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAM9C,iBACJiB,OACAG,UACAqB,KAAKkB,GAAG,CAAC,CAACC,IAAMA,EAAEd,IAAI;IAGxB,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAMe,SAAShE,SAAS,oBAAoB4C,KAAKb,MAAM;IACvD,IAAK,IAAIkC,IAAI,GAAGA,IAAIrB,KAAKb,MAAM,EAAEkC,KAAK1C,SAAS2C,QAAQ,CAAE;QACvD,MAAMC,QAAQvB,KAAKwB,KAAK,CAACH,GAAGA,IAAI1C,SAAS2C,QAAQ;QACjD,MAAMzC,UAAU,MAAMF,SAAS8C,cAAc,CAACF,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEd,IAAI;QACrE,MAAM7B,MAAMK,OAAO,CAAC6C,YAAY,CAC9BH,MAAML,GAAG,CAAC,CAACS,KAAKR;YACd,MAAM,EAAErD,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQmB,OAAO,CAACsC,EAAE,EAAE/B,WAAW;YACpD,MAAMwC,IAAI,IAAIC,UAAUzC;YACxB,IAAK,IAAInB,IAAI,GAAGA,IAAImB,WAAWnB,IAAK2D,CAAC,CAAC3D,EAAE,GAAGC,KAAK4D,KAAK,CAAChE,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEc,MAAMyC,IAAIzC,IAAI;gBAAES,OAAOgC,IAAIhC,KAAK;gBAAEvB;gBAAO2D,QAAQlB,OAAOmB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;QAEFb,OAAOc,IAAI,CAAChE,KAAKmB,GAAG,CAACgC,IAAI1C,SAAS2C,QAAQ,EAAEtB,KAAKb,MAAM;IACzD;IACAiC,OAAOe,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB5D,KAAY,EAAE6D,IAAY,EAAE3D,OAA4B;IACrG,MAAMI,UAAU,AAAC,CAAA,MAAMN,MAAMK,OAAO,CAACC,OAAO,EAAC,EAAGC,MAAM,CAAC,CAACC,MAAQN,QAAQO,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIJ,QAAQK,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIjC,WAAW,mBAAmB,GAAG4B,QAAQK,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAemD,mBAAmB9D,KAAY,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAE9D,OAAqB;QACrG,cAIF6D;IAJd,MAAME,WAAU,eAAA,AAAChE,IAAgCgE,OAAO,cAAxC,0BAAA,eAA4C,AAAChE,IAAgCiE,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIvF,WAAW,eAAe;IAClD,MAAMyB,WAAW,MAAMnB,YAAYiB;IACnC,MAAMW,YAAYlB,KAAKmB,GAAG,CAAC5B,YAAYkB,SAASf,IAAI;IACpD,MAAMyC,OAAO,EAACkC,eAAAA,MAAMI,KAAK,CAAC,gCAAZJ,0BAAAA,eAAkC,EAAE,EAAExD,MAAM,CAAC,CAAC6D,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIE,IAAI,CAAC;IAClH,MAAM,EAAEhF,GAAGiF,EAAE,EAAE,GAAGrF,QAAQ,MAAMiB,SAASqE,UAAU,CAAC3C,OAAOjB,WAAW;IAEtE,OAAOZ,MAAMK,OAAO,CAACoE,UAAU,CAACF,IAAI3D,WAAWoD,OAAO9D;AACxD;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAewE,aAAa1E,KAAY,EAAEU,IAAY;IAC3D,OAAOV,MAAMK,OAAO,CAACsE,SAAS,CAACjE;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAekE,aAAa5E,KAAY,EAAE6D,IAAY,EAAEnD,IAAY,EAAEmE,IAAgE;IAC3I,OAAO7E,MAAMK,OAAO,CAACyE,OAAO,CAACpE,MAAMmE;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}

package/dist/esm/embed/registry.d.ts

@@ -1,3 +1,3 @@
 import type { Config } from '../config/index.js';
-import type { EmbedProvider } from './types.js';
-export declare function getProvider(cfg: Config): Promise<EmbedProvider>;
+import type { EmbedCallOptions, EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config, options?: EmbedCallOptions): Promise<EmbedProvider>;

package/dist/esm/embed/registry.js

@@ -1,6 +1,7 @@
 import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { cohereProvider } from './cohere.js';
+import { effectiveEmbedEndpoint } from './endpoint.js';
 import { openaiProvider } from './openai.js';
 import { staticProvider } from './static.js';
 // One factory per wire protocol: adding a protocol is a new file plus an entry here, not
@@ -11,24 +12,52 @@ const REGISTRY = {
     cohere: (model, url, key)=>cohereProvider(model, url, key)
 };
 const providers = new Map();
-export function getProvider(cfg) {
-    var _e_url, _e_key, _e_languages;
+function aborted(reason) {
+    return Promise.reject(reason);
+}
+function waitForProvider(provider, signal) {
+    if (signal.aborted) return aborted(signal.reason);
+    return new Promise((resolve, reject)=>{
+        const cleanup = ()=>signal.removeEventListener('abort', onAbort);
+        const onAbort = ()=>{
+            cleanup();
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', onAbort, {
+            once: true
+        });
+        provider.then((value)=>{
+            cleanup();
+            resolve(value);
+        }, (err)=>{
+            cleanup();
+            reject(err);
+        });
+        if (signal.aborted) onAbort();
+    });
+}
+export function getProvider(cfg, options = {}) {
+    var _e_key, _e_languages;
+    var _options_signal;
+    // A caller that has already left must not start shared model initialization.
+    if ((_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.aborted) return aborted(options.signal.reason);
     const e = embedConfig(cfg);
     if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');
     const factory = REGISTRY[e.provider];
     if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider "${e.provider}" has no provider implementation in this build`);
+    const endpoint = effectiveEmbedEndpoint(e.provider, e.url);
     // Every input that shapes the provider is in the key: two trees sharing an endpoint and
     // model but declaring different languages or key env vars are two providers, not one.
     const sig = JSON.stringify([
         e.provider,
         e.model,
-        (_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '',
+        endpoint !== null && endpoint !== void 0 ? endpoint : '',
         (_e_key = e.key) !== null && _e_key !== void 0 ? _e_key : '',
         (_e_languages = e.languages) !== null && _e_languages !== void 0 ? _e_languages : []
     ]);
     let p = providers.get(sig);
     if (!p) {
-        p = factory(e.model, e.url, e.key);
+        p = factory(e.model, endpoint, e.key);
         // Owner-declared languages override a provider's own: the config is the one home for
         // what no model card can state (http models have no fetchable card here).
         if (e.languages) p = p.then((prov)=>({
@@ -40,5 +69,5 @@ export function getProvider(cfg) {
         // poisoning this config for the process lifetime.
         p.catch(()=>providers.delete(sig));
     }
-    return p;
+    return options.signal ? waitForProvider(p, options.signal) : p;
 }

package/dist/esm/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","getProvider","cfg","e","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQR,eAAeK,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,OAAO,SAASC,YAAYC,GAAW;QAOYC,QAAaA,QAAaA;IAN3E,MAAMA,IAAIhB,YAAYe;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAC/C,MAAMgB,UAAUZ,QAAQ,CAACW,EAAEE,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAIhB,WAAW,eAAe,CAAC,gBAAgB,EAAEe,EAAEE,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,wFAAwF;IACxF,sFAAsF;IACtF,MAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEE,QAAQ;QAAEF,EAAET,KAAK;SAAES,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,SAAAA,EAAEN,GAAG,cAALM,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIO,IAAIX,UAAUY,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAET,KAAK,EAAES,EAAEP,GAAG,EAAEO,EAAEN,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIM,EAAEM,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,CAACC,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEJ,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzEV,UAAUe,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC,IAAMhB,UAAUiB,MAAM,CAACV;IACjC;IACA,OAAOI;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction aborted(reason: unknown): Promise<never> {\n  return Promise.reject(reason);\n}\n\nfunction waitForProvider(provider: Promise<EmbedProvider>, signal: AbortSignal): Promise<EmbedProvider> {\n  if (signal.aborted) return aborted(signal.reason);\n  return new Promise((resolve, reject) => {\n    const cleanup = () => signal.removeEventListener('abort', onAbort);\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    provider.then(\n      (value) => {\n        cleanup();\n        resolve(value);\n      },\n      (err: unknown) => {\n        cleanup();\n        reject(err);\n      }\n    );\n    if (signal.aborted) onAbort();\n  });\n}\n\nexport function getProvider(cfg: Config, options: EmbedCallOptions = {}): Promise<EmbedProvider> {\n  // A caller that has already left must not start shared model initialization.\n  if (options.signal?.aborted) return aborted(options.signal.reason);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  const endpoint = effectiveEmbedEndpoint(e.provider, e.url);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, endpoint ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, endpoint, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return options.signal ? waitForProvider(p, options.signal) : p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","effectiveEmbedEndpoint","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","aborted","reason","Promise","reject","waitForProvider","provider","signal","resolve","cleanup","removeEventListener","onAbort","addEventListener","once","then","value","err","getProvider","cfg","options","e","factory","endpoint","sig","JSON","stringify","languages","p","get","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQT,eAAeM,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,SAASC,QAAQC,MAAe;IAC9B,OAAOC,QAAQC,MAAM,CAACF;AACxB;AAEA,SAASG,gBAAgBC,QAAgC,EAAEC,MAAmB;IAC5E,IAAIA,OAAON,OAAO,EAAE,OAAOA,QAAQM,OAAOL,MAAM;IAChD,OAAO,IAAIC,QAAQ,CAACK,SAASJ;QAC3B,MAAMK,UAAU,IAAMF,OAAOG,mBAAmB,CAAC,SAASC;QAC1D,MAAMA,UAAU;YACdF;YACAL,OAAOG,OAAOL,MAAM;QACtB;QACAK,OAAOK,gBAAgB,CAAC,SAASD,SAAS;YAAEE,MAAM;QAAK;QACvDP,SAASQ,IAAI,CACX,CAACC;YACCN;YACAD,QAAQO;QACV,GACA,CAACC;YACCP;YACAL,OAAOY;QACT;QAEF,IAAIT,OAAON,OAAO,EAAEU;IACtB;AACF;AAEA,OAAO,SAASM,YAAYC,GAAW,EAAEC,UAA4B,CAAC,CAAC;QAUJC,QAAaA;QAR1ED;IADJ,6EAA6E;IAC7E,KAAIA,kBAAAA,QAAQZ,MAAM,cAAdY,sCAAAA,gBAAgBlB,OAAO,EAAE,OAAOA,QAAQkB,QAAQZ,MAAM,CAACL,MAAM;IACjE,MAAMkB,IAAIlC,YAAYgC;IACtB,IAAI,CAACE,GAAG,MAAM,IAAIjC,WAAW,kBAAkB;IAC/C,MAAMkC,UAAU7B,QAAQ,CAAC4B,EAAEd,QAAQ,CAAC;IACpC,IAAI,CAACe,SAAS,MAAM,IAAIlC,WAAW,eAAe,CAAC,gBAAgB,EAAEiC,EAAEd,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,MAAMgB,WAAWjC,uBAAuB+B,EAAEd,QAAQ,EAAEc,EAAExB,GAAG;IACzD,wFAAwF;IACxF,sFAAsF;IACtF,MAAM2B,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEd,QAAQ;QAAEc,EAAE1B,KAAK;QAAE4B,qBAAAA,sBAAAA,WAAY;SAAIF,SAAAA,EAAEvB,GAAG,cAALuB,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAChG,IAAIO,IAAI5B,UAAU6B,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAE1B,KAAK,EAAE4B,UAAUF,EAAEvB,GAAG;QACpC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIuB,EAAEM,SAAS,EAAEC,IAAIA,EAAEb,IAAI,CAAC,CAACe,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEH,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzE3B,UAAU+B,GAAG,CAACP,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEI,KAAK,CAAC,IAAMhC,UAAUiC,MAAM,CAACT;IACjC;IACA,OAAOJ,QAAQZ,MAAM,GAAGF,gBAAgBsB,GAAGR,QAAQZ,MAAM,IAAIoB;AAC/D"}

package/dist/esm/embed/static.js

@@ -68,7 +68,15 @@ export async function staticProvider(model, root) {
         dims,
         batchCap: BATCH_CAP,
         languages,
-        embedDocuments: async (texts)=>texts.map(one),
-        embedQuery: async (text)=>one(text)
+        embedDocuments: async (texts, options)=>{
+            var _options_signal;
+            options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            return texts.map(one);
+        },
+        embedQuery: async (text, options)=>{
+            var _options_signal;
+            options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            return one(text);
+        }
     };
 }

package/dist/esm/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QAAEF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QAAE4B;QAAM4B,UAAU9D;QAAWU;QAAWqD,gBAAgB,OAAOC,QAAUA,MAAMC,GAAG,CAACjB;QAAMkB,YAAY,OAAOjB,OAASD,IAAIC;IAAM;AAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return {\n    id: `static:${model}`,\n    dims,\n    batchCap: BATCH_CAP,\n    languages,\n    embedDocuments: async (texts: string[], options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return texts.map(one);\n    },\n    embedQuery: async (text: string, options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return one(text);\n    },\n  };\n}\n"],"names":["existsSync","readFileSync","Module","join","SenseError","downloadModel","isDownloadable","MODEL_FILENAMES","MODEL_FILES","modelDir","readLanguages","BATCH_CAP","_require","require","createRequire","url","staticProvider","model","root","tokenizerJson","dir","languages","file","into","console","error","cached","length","undefined","raw","headerLen","Number","readBigUInt64LE","header","JSON","parse","subarray","toString","entry","Object","entries","find","k","dtype","spec","dims","shape","dataStart","byteOffset","data_offsets","matrix","Float32Array","buffer","slice","Tokenizer","tok","unkId","vocab","unk_token","one","text","ids","encode","add_special_tokens","filter","id","isInteger","v","off","d","norm","Math","sqrt","batchCap","embedDocuments","texts","options","signal","throwIfAborted","map","embedQuery"],"mappings":"AAAA,SAASA,UAAU,EAAEC,YAAY,QAAQ,UAAU;AACnD,OAAOC,YAAY,cAAc;AACjC,SAASC,IAAI,QAAQ,YAAY;AACjC,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,aAAa,EAAEC,cAAc,EAAEC,eAAe,EAAEC,WAAW,EAAEC,QAAQ,EAAEC,aAAa,QAAQ,aAAa;AAGlH,MAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,MAAMC,WAAW,OAAOC,YAAY,cAAcX,OAAOY,aAAa,CAAC,YAAYC,GAAG,IAAIF;AAE1F,2FAA2F;AAC3F,yDAAyD;AACzD,OAAO,eAAeG,eAAeC,KAAa,EAAEC,IAAa;;QA+BpBC,sBAA7BA,4BAAAA;IA9Bd,IAAIC;IACJ,IAAIC;IACJ,IAAIf,eAAeW,QAAQ;QACzB,qFAAqF;QACrFG,MAAM,MAAMf,cAAcY,OAAO,CAACK,MAAMC,OAASC,QAAQC,KAAK,CAAC,CAAC,gBAAgB,EAAER,MAAM,CAAC,EAAEK,KAAK,MAAM,EAAEC,MAAM,GAAGL;QACjH,uFAAuF;QACvF,MAAMQ,SAAShB,cAAcO,OAAOC;QACpCG,YAAYK,UAAUA,OAAOC,MAAM,GAAG,IAAID,SAASE;IACrD,OAAO;QACLR,MAAMX,SAASQ,OAAOC;QACtB,KAAK,MAAMI,QAAQd,YAAa;YAC9B,IAAI,CAACR,WAAWG,KAAKiB,KAAKE,QAAQ;gBAChC,MAAM,IAAIlB,WAAW,uBAAuB,CAAC,YAAY,EAAEa,MAAM,6BAA6B,EAAEG,IAAI,YAAY,EAAEb,gBAAgB,kBAAkB,CAAC;YACvJ;QACF;IACF;IAEA,MAAMsB,MAAM5B,aAAaE,KAAKiB,KAAK;IACnC,MAAMU,YAAYC,OAAOF,IAAIG,eAAe,CAAC;IAC7C,MAAMC,SAASC,KAAKC,KAAK,CAACN,IAAIO,QAAQ,CAAC,GAAG,IAAIN,WAAWO,QAAQ,CAAC;IAClE,MAAMC,QAAQC,OAAOC,OAAO,CAACP,QAAQQ,IAAI,CAAC,CAAC,CAACC,EAAE,GAAKA,MAAM;IACzD,IAAI,CAACJ,SAASA,KAAK,CAAC,EAAE,CAACK,KAAK,KAAK,OAAO,MAAM,IAAIvC,WAAW,eAAe,GAAGa,MAAM,oCAAoC,CAAC;IAC1H,MAAM2B,OAAON,KAAK,CAAC,EAAE;IACrB,MAAMO,OAAOD,KAAKE,KAAK,CAAC,EAAE;IAC1B,MAAMC,YAAYlB,IAAImB,UAAU,GAAG,IAAIlB,YAAYc,KAAKK,YAAY,CAAC,EAAE;IACvE,MAAMC,SAASH,YAAY,MAAM,IAAI,IAAII,aAAatB,IAAIuB,MAAM,EAAEL,WAAWH,KAAKE,KAAK,CAAC,EAAE,GAAGD,QAAQ,IAAIM,aAAatB,IAAIuB,MAAM,CAACC,KAAK,CAACN,WAAWA,YAAYH,KAAKE,KAAK,CAAC,EAAE,GAAGD,OAAO;IAErL,MAAM1B,gBAAgBe,KAAKC,KAAK,CAAClC,aAAaE,KAAKiB,KAAK,mBAAmB;IAC3E,MAAM,EAAEkC,SAAS,EAAE,GAAG1C,SAAS;IAC/B,MAAM2C,MAAM,IAAID,UAAUnC,eAAe,CAAC;IAC1C,MAAMqC,iBAAQrC,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqBsC,KAAK,cAA1BtC,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqBuC,SAAS,CAAC,uCAAI,CAAC;IAE/E,SAASC,IAAIC,IAAY;QACvB,0FAA0F;QAC1F,0FAA0F;QAC1F,MAAMC,MAAM,AAACN,IAAIO,MAAM,CAACF,MAAM;YAAEG,oBAAoB;QAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,CAACC,KAAOlC,OAAOmC,SAAS,CAACD,OAAOA,OAAOT;QAC5H,MAAMW,IAAI,IAAIhB,aAAaN;QAC3B,IAAIgB,IAAIlC,MAAM,KAAK,GAAG,OAAOwC;QAC7B,KAAK,MAAMF,MAAMJ,IAAK;YACpB,MAAMO,MAAMH,KAAKpB;YACjB,IAAK,IAAIwB,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAInB,MAAM,CAACkB,MAAMC,EAAE;QACxD;QACA,IAAIC,OAAO;QACX,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAK;YAC7BF,CAAC,CAACE,EAAE,IAAIR,IAAIlC,MAAM;YAClB2C,QAAQH,CAAC,CAACE,EAAE,GAAGF,CAAC,CAACE,EAAE;QACrB;QACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;QACzB,IAAK,IAAID,IAAI,GAAGA,IAAIxB,MAAMwB,IAAKF,CAAC,CAACE,EAAE,IAAIC;QACvC,OAAOH;IACT;IAEA,mEAAmE;IACnE,OAAO;QACLF,IAAI,CAAC,OAAO,EAAEhD,OAAO;QACrB4B;QACA4B,UAAU9D;QACVU;QACAqD,gBAAgB,OAAOC,OAAiBC;gBACtCA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOH,MAAMI,GAAG,CAACpB;QACnB;QACAqB,YAAY,OAAOpB,MAAcgB;gBAC/BA;YAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;YAC/B,OAAOnB,IAAIC;QACb;IACF;AACF"}

package/dist/esm/embed/store.js

@@ -3,7 +3,7 @@ import { join } from 'node:path';
 import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { writeFileAtomic } from '../lib/atomic-write.js';
-import { fetchWithRetry } from './http.js';
+import { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.js';
 import { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.js';
 import { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.js';
 // identity.ts holds paths and cache identity, no network; store.ts holds the network fetches
@@ -29,20 +29,28 @@ async function resolveSha(model, root = DEFAULT_MODELS_ROOT) {
     const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;
     // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN
     // redirect would read the CDN's headers instead, which do not carry it.
-    let res;
-    try {
-        res = await fetchWithRetry(url, {
-            method: 'HEAD',
-            redirect: 'manual'
-        });
-    } catch (err) {
-        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
-    }
+    const res = await fetchWithRetry(url, {
+        method: 'HEAD',
+        redirect: 'manual'
+    }, {
+        attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
+    });
     const sha = res.headers.get('x-repo-commit');
-    if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);
+    if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);
     writeRef(model, sha, root);
     return sha;
 }
+function isStringArray(value) {
+    return Array.isArray(value) && value.every((part)=>typeof part === 'string');
+}
+function isHfModelInfo(value) {
+    if (value === null || typeof value !== 'object') return false;
+    if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;
+    if (!('cardData' in value) || value.cardData === undefined) return true;
+    if (value.cardData === null || typeof value.cardData !== 'object') return false;
+    if (!('language' in value.cardData) || value.cardData.language === undefined) return true;
+    return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);
+}
 // cardData.language is the structured field; tags are a fallback for cards that only tag
 // languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.
 function languagesFromCard(info) {
@@ -72,7 +80,9 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     }
     let res;
     try {
-        res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});
+        res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, {
+            attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
+        });
     } catch  {
         cardUnreadable(model);
         return;
@@ -83,7 +93,9 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     }
     let info;
     try {
-        info = await res.json();
+        const decoded = JSON.parse(new TextDecoder().decode(res.body));
+        if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');
+        info = decoded;
     } catch  {
         cardUnreadable(model);
         return;
@@ -91,14 +103,11 @@ async function ensureLanguages(model, root = DEFAULT_MODELS_ROOT) {
     writeLanguages(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : [], root);
 }
 async function fetchToFile(url, dest) {
-    let res;
-    try {
-        res = await fetchWithRetry(url, {});
-    } catch (err) {
-        throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${err.message}`);
-    }
-    if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);
-    writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));
+    const res = await fetchWithRetry(url, {}, {
+        attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS
+    });
+    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);
+    writeFileAtomic(dest, Buffer.from(res.body));
 }
 // Fetches an already-known sha straight into its snapshot dir, without touching refs/main --
 // for downloadModel below, and for callers pinning an exact revision (the parity suite).

package/dist/esm/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","err","message","sha","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","undefined","cardUnreadable","console","error","ensureLanguages","measured","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAASC,cAAc,QAAQ,YAAY;AAC3C,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIrB,YAAYoB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,IAAIM;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK;YAAEE,QAAQ;YAAQC,UAAU;QAAS;IACvE,EAAE,OAAOC,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,MAAMC,MAAML,IAAIM,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIjC,WAAW,eAAe,GAAG2B,IAAI,0CAA0C,CAAC;IAChGhB,SAASW,OAAOW,KAAKR;IACrB,OAAOQ;AACT;AAOA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASG,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,MAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAAClC;IAC1C,OAAOgC,SAASD,MAAM,GAAG,IAAIC,WAAWG;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASC,eAAe1B,KAAa;IACnC2B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAE5B,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAe6B,gBAAgB7B,KAAa,EAAEG,OAAetB,mBAAmB;QA0BxDiC;IAzBtB,IAAIxC,WAAWU,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM2B,WAAWvC,wBAAwB,CAACS,MAAM;IAChD,IAAI8B,UAAU;QACZ1C,eAAeY,OAAO8B,UAAU3B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAe,CAAC,kCAAkC,EAAEoB,OAAO,EAAE,CAAC;IAC5E,EAAE,OAAM;QACN0B,eAAe1B;QACf;IACF;IACA,IAAI,CAACM,IAAIyB,EAAE,EAAE;QACXL,eAAe1B;QACf;IACF;IACA,IAAIe;IACJ,IAAI;QACFA,OAAQ,MAAMT,IAAI0B,IAAI;IACxB,EAAE,OAAM;QACNN,eAAe1B;QACf;IACF;IACAZ,eAAeY,QAAOc,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAEX;AACvD;AAEA,eAAe8B,YAAY5B,GAAW,EAAE6B,IAAY;IAClD,IAAI5B;IACJ,IAAI;QACFA,MAAM,MAAM1B,eAAeyB,KAAK,CAAC;IACnC,EAAE,OAAOI,KAAK;QACZ,MAAM,IAAI/B,WAAW,eAAe,CAAC,gBAAgB,EAAE2B,IAAI,EAAE,EAAE,AAACI,IAAcC,OAAO,EAAE;IACzF;IACA,IAAI,CAACJ,IAAIyB,EAAE,EAAE,MAAM,IAAIrD,WAAW,eAAe,GAAG2B,IAAI,SAAS,EAAEC,IAAI6B,MAAM,EAAE;IAC/ExD,gBAAgBuD,MAAME,OAAOC,IAAI,CAAC,MAAM/B,IAAIgC,WAAW;AACzD;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeC,sBAAsBvC,KAAa,EAAEW,GAAW,EAAE6B,MAA4C,EAAErC,OAAetB,mBAAmB;IACtJ,MAAM4D,MAAMtD,YAAYa,OAAOW,KAAKR;IACpC5B,UAAUkE,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQ1D,YAAa;QAC9B,IAAIX,WAAWE,KAAKiE,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMR,YAAY,CAAC,uBAAuB,EAAEjC,MAAM,SAAS,EAAEW,IAAI,CAAC,EAAEgC,MAAM,EAAEnE,KAAKiE,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAc5C,KAAa,EAAEwC,MAA4C,EAAErC,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAItB,WAAW,eAAe,CAAC,aAAa,EAAEsB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMW,MAAM,MAAMT,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM0B,gBAAgB7B,OAAOG;IAC7B,OAAOoC,sBAAsBvC,OAAOW,KAAK6B,QAAQrC;AACnD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { type BufferedResponse, fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  const res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' }, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((part) => typeof part === 'string');\n}\n\nfunction isHfModelInfo(value: unknown): value is HfModelInfo {\n  if (value === null || typeof value !== 'object') return false;\n  if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;\n  if (!('cardData' in value) || value.cardData === undefined) return true;\n  if (value.cardData === null || typeof value.cardData !== 'object') return false;\n  if (!('language' in value.cardData) || value.cardData.language === undefined) return true;\n  return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: BufferedResponse;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    const decoded: unknown = JSON.parse(new TextDecoder().decode(res.body));\n    if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');\n    info = decoded;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  const res = await fetchWithRetry(url, {}, { attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS });\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(res.body));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["existsSync","mkdirSync","join","embedConfig","SenseError","writeFileAtomic","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","DEFAULT_MODELS_ROOT","hasModelFiles","isDownloadable","languagesPath","MODEL_FILES","readRef","snapshotDir","writeLanguages","writeRef","isKnownLanguageTag","MEASURED_MODEL_LANGUAGES","MODEL_FILENAMES","modelDir","readLanguages","refsMainPath","modelPresent","cfg","e","provider","model","localModelMissing","resolveSha","root","existing","url","res","method","redirect","attemptTimeoutMs","sha","headers","get","isStringArray","value","Array","isArray","every","part","isHfModelInfo","tags","undefined","cardData","language","languagesFromCard","info","lang","length","tagLangs","filter","cardUnreadable","console","error","ensureLanguages","measured","ok","decoded","JSON","parse","TextDecoder","decode","body","Error","fetchToFile","dest","status","Buffer","from","downloadModelRevision","onFile","dir","recursive","file","downloadModel"],"mappings":"AAAA,SAASA,UAAU,EAAEC,SAAS,QAAQ,UAAU;AAChD,SAASC,IAAI,QAAQ,YAAY;AAEjC,SAASC,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,eAAe,QAAQ,yBAAyB;AACzD,SAAgCC,cAAc,EAAEC,uBAAuB,EAAEC,6BAA6B,QAAQ,YAAY;AAC1H,SAASC,mBAAmB,EAAEC,aAAa,EAAEC,cAAc,EAAEC,aAAa,EAAEC,WAAW,EAAEC,OAAO,EAAEC,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAC/J,SAASC,kBAAkB,EAAEC,wBAAwB,QAAQ,iBAAiB;AAE9E,6FAA6F;AAC7F,4FAA4F;AAC5F,SAAST,aAAa,EAAEC,cAAc,EAAES,eAAe,EAAEP,WAAW,EAAEQ,QAAQ,EAAEC,aAAa,EAAEC,YAAY,EAAER,WAAW,EAAEC,cAAc,EAAEC,QAAQ,QAAQ,gBAAgB;AAE1K,6FAA6F;AAC7F,0CAA0C;AAC1C,OAAO,SAASO,aAAaC,GAAW;IACtC,MAAMC,IAAIvB,YAAYsB;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEC,QAAQ,KAAK,YAAYjB,cAAcgB,EAAEE,KAAK;AACzD;AAIA,6FAA6F;AAC7F,4EAA4E;AAC5E,OAAO,SAASC,kBAAkBH,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEC,QAAQ,KAAK,YAAY,CAAChB,eAAee,EAAEE,KAAK,KAAK,CAAClB,cAAcgB,EAAEE,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,eAAeE,WAAWF,KAAa,EAAEG,OAAetB,mBAAmB;IACzE,MAAMuB,WAAWlB,QAAQc,OAAOG;IAChC,IAAIC,UAAU,OAAOA;IACrB,MAAMC,MAAM,CAAC,uBAAuB,EAAEL,MAAM,+BAA+B,CAAC;IAC5E,qFAAqF;IACrF,wEAAwE;IACxE,MAAMM,MAAM,MAAM5B,eAAe2B,KAAK;QAAEE,QAAQ;QAAQC,UAAU;IAAS,GAAG;QAAEC,kBAAkB9B;IAAwB;IAC1H,MAAM+B,MAAMJ,IAAIK,OAAO,CAACC,GAAG,CAAC;IAC5B,IAAI,CAACF,KAAK,MAAM,IAAIlC,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,0CAA0C,CAAC;IACpGhB,SAASW,OAAOU,KAAKP;IACrB,OAAOO;AACT;AAOA,SAASG,cAAcC,KAAc;IACnC,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS;AACvE;AAEA,SAASC,cAAcL,KAAc;IACnC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,UAAU,OAAO;IACxD,IAAI,UAAUA,SAASA,MAAMM,IAAI,KAAKC,aAAa,CAACR,cAAcC,MAAMM,IAAI,GAAG,OAAO;IACtF,IAAI,CAAE,CAAA,cAAcN,KAAI,KAAMA,MAAMQ,QAAQ,KAAKD,WAAW,OAAO;IACnE,IAAIP,MAAMQ,QAAQ,KAAK,QAAQ,OAAOR,MAAMQ,QAAQ,KAAK,UAAU,OAAO;IAC1E,IAAI,CAAE,CAAA,cAAcR,MAAMQ,QAAQ,AAAD,KAAMR,MAAMQ,QAAQ,CAACC,QAAQ,KAAKF,WAAW,OAAO;IACrF,OAAO,OAAOP,MAAMQ,QAAQ,CAACC,QAAQ,KAAK,YAAYV,cAAcC,MAAMQ,QAAQ,CAACC,QAAQ;AAC7F;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASC,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,MAAMC,QAAOD,iBAAAA,KAAKH,QAAQ,cAAbG,qCAAAA,eAAeF,QAAQ;IACpC,IAAI,OAAOG,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIX,MAAMC,OAAO,CAACU,SAASA,KAAKC,MAAM,GAAG,GAAG,OAAOD;IACnD,MAAME,WAAW,EAACH,aAAAA,KAAKL,IAAI,cAATK,wBAAAA,aAAa,EAAE,EAAEI,MAAM,CAACvC;IAC1C,OAAOsC,SAASD,MAAM,GAAG,IAAIC,WAAWP;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASS,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,CAAC,yCAAyC,EAAEhC,MAAM,kGAAkG,CAAC;AACrK;AAEA,eAAeiC,gBAAgBjC,KAAa,EAAEG,OAAetB,mBAAmB;QA4BxD2C;IA3BtB,IAAIpD,WAAWY,cAAcgB,OAAOG,QAAQ;IAC5C,kFAAkF;IAClF,MAAM+B,WAAW3C,wBAAwB,CAACS,MAAM;IAChD,IAAIkC,UAAU;QACZ9C,eAAeY,OAAOkC,UAAU/B;QAChC;IACF;IACA,IAAIG;IACJ,IAAI;QACFA,MAAM,MAAM5B,eAAe,CAAC,kCAAkC,EAAEsB,OAAO,EAAE,CAAC,GAAG;YAAES,kBAAkB9B;QAAwB;IAC3H,EAAE,OAAM;QACNmD,eAAe9B;QACf;IACF;IACA,IAAI,CAACM,IAAI6B,EAAE,EAAE;QACXL,eAAe9B;QACf;IACF;IACA,IAAIyB;IACJ,IAAI;QACF,MAAMW,UAAmBC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAAClC,IAAImC,IAAI;QACrE,IAAI,CAACtB,cAAciB,UAAU,MAAM,IAAIM,MAAM;QAC7CjB,OAAOW;IACT,EAAE,OAAM;QACNN,eAAe9B;QACf;IACF;IACAZ,eAAeY,QAAOwB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,qBAA2B,EAAE,EAAErB;AACvD;AAEA,eAAewC,YAAYtC,GAAW,EAAEuC,IAAY;IAClD,MAAMtC,MAAM,MAAM5B,eAAe2B,KAAK,CAAC,GAAG;QAAEI,kBAAkB7B;IAA8B;IAC5F,IAAI,CAAC0B,IAAI6B,EAAE,EAAE,MAAM,IAAI3D,WAAW,eAAe,GAAG8B,IAAID,GAAG,CAAC,SAAS,EAAEC,IAAIuC,MAAM,EAAE;IACnFpE,gBAAgBmE,MAAME,OAAOC,IAAI,CAACzC,IAAImC,IAAI;AAC5C;AAEA,6FAA6F;AAC7F,yFAAyF;AACzF,OAAO,eAAeO,sBAAsBhD,KAAa,EAAEU,GAAW,EAAEuC,MAA4C,EAAE9C,OAAetB,mBAAmB;IACtJ,MAAMqE,MAAM/D,YAAYa,OAAOU,KAAKP;IACpC9B,UAAU6E,KAAK;QAAEC,WAAW;IAAK;IACjC,KAAK,MAAMC,QAAQnE,YAAa;QAC9B,IAAIb,WAAWE,KAAK4E,KAAKE,QAAQ;QACjCH,mBAAAA,6BAAAA,OAASG,MAAMF;QACf,MAAMP,YAAY,CAAC,uBAAuB,EAAE3C,MAAM,SAAS,EAAEU,IAAI,CAAC,EAAE0C,MAAM,EAAE9E,KAAK4E,KAAKE;IACxF;IACA,OAAOF;AACT;AAEA,2FAA2F;AAC3F,2EAA2E;AAC3E,OAAO,eAAeG,cAAcrD,KAAa,EAAEiD,MAA4C,EAAE9C,OAAetB,mBAAmB;IACjI,IAAI,CAACE,eAAeiB,QAAQ;QAC1B,MAAM,IAAIxB,WAAW,eAAe,CAAC,aAAa,EAAEwB,MAAM,qMAAqM,CAAC;IAClQ;IACA,MAAMU,MAAM,MAAMR,WAAWF,OAAOG;IACpC,sFAAsF;IACtF,oFAAoF;IACpF,MAAM8B,gBAAgBjC,OAAOG;IAC7B,OAAO6C,sBAAsBhD,OAAOU,KAAKuC,QAAQ9C;AACnD"}

package/dist/esm/embed/types.d.ts

@@ -1,9 +1,12 @@
 export declare const STORE_DIMS = 256;
+export interface EmbedCallOptions {
+    signal?: AbortSignal;
+}
 export interface EmbedProvider {
     id: string;
     dims: number;
     batchCap: number;
     languages?: string[];
-    embedDocuments(texts: string[]): Promise<Float32Array[]>;
-    embedQuery(text: string): Promise<Float32Array>;
+    embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;
+    embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;
 }

package/dist/esm/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\nexport interface EmbedCallOptions {\n  signal?: AbortSignal;\n}\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;\n  embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;AAClG,OAAO,MAAMA,aAAa,IAAI"}

package/dist/esm/features/embed.js

@@ -1,9 +1,11 @@
 import { CHUNK_VERSION, chunkFromBlocks, parse } from '../chunk/index.js';
 import { embedConfig } from '../config/index.js';
-import { stashChunkText } from '../embed/handoff.js';
+import { embedEndpointIdentity } from '../embed/endpoint.js';
+import { identifyChunkText, stashChunkText } from '../embed/handoff.js';
 import { modelIdentity } from '../embed/identity.js';
 import { countLines } from '../scan/frontmatter.js';
 import { appendRows } from '../store/shared.js';
+const CHUNK_LOOKUP_PAGE = 512;
 // The title/summary prefix mirrors bm25 field weighting. Chunks the body, not the raw file, so
 // `offset` shifts line numbers back onto raw and a range stays a direct Read range.
 function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
@@ -24,7 +26,7 @@ function chunksOf(blocks, body, search, offset = 0, chunkTokens) {
 export const embed = {
     name: 'embed',
     async schema (db) {
-        await db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
+        await db.exec(`CREATE TABLE IF NOT EXISTS embeddings ("path" TEXT, chunk INTEGER, start_line INTEGER, end_line INTEGER, content_identity TEXT NOT NULL, scale REAL, vector BLOB, PRIMARY KEY ("path", chunk))`);
     },
     extract (raw, body, search, _data, cfg, blocks) {
         var _cfg_embed;
@@ -32,39 +34,96 @@ export const embed = {
         // blocks comes from parseFile's shared parse; falls back to parsing body for a direct call.
         return chunksOf(blocks !== null && blocks !== void 0 ? blocks : parse(body), body, search, countLines(raw) - countLines(body), cfg === null || cfg === void 0 ? void 0 : (_cfg_embed = cfg.embed) === null || _cfg_embed === void 0 ? void 0 : _cfg_embed.chunkTokens);
     },
-    async remove (db, paths) {
-        if (paths.length === 0) return;
-        await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', paths.map((p)=>[
-                p
+    async remove (db, paths, delta) {
+        const vanishedSet = new Set(delta.vanished);
+        const vanished = paths.filter((path)=>vanishedSet.has(path));
+        if (vanished.length === 0) return;
+        await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', vanished.map((path)=>[
+                path
             ]));
     },
     // A tree with no embedding model never had extract() run for the doc (reconcile.ts's per-file
     // filter skips it), so extracted is undefined here -- those docs contribute no rows.
-    async store (db, docs) {
+    async store (db, docs, delta) {
         const rows = [];
         const texts = new Map();
+        const added = new Set(delta.added);
+        // Keyset pages bound comparison metadata without reading native vector payloads.
+        let firstPage;
+        let nextPage;
         for (const { path, extracted } of docs){
-            if (!extracted) continue;
-            const chunks = extracted;
-            chunks.forEach((c, idx)=>rows.push([
+            const chunks = extracted !== null && extracted !== void 0 ? extracted : [];
+            if (chunks.length === 0) {
+                if (!added.has(path)) await db.runBatch('DELETE FROM embeddings WHERE "path" = ?', [
+                    [
+                        path
+                    ]
+                ]);
+                continue;
+            }
+            const identified = chunks.map((chunk)=>identifyChunkText(chunk.text));
+            const kept = new Set();
+            if (!added.has(path)) {
+                let cursor;
+                for(;;){
+                    let page;
+                    if (cursor === undefined) {
+                        firstPage !== null && firstPage !== void 0 ? firstPage : firstPage = await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);
+                        page = await firstPage.all(path);
+                    } else {
+                        nextPage !== null && nextPage !== void 0 ? nextPage : nextPage = await db.prepare(`SELECT chunk, start_line, end_line, content_identity FROM embeddings WHERE "path" = ? AND chunk > ? ORDER BY chunk LIMIT ${CHUNK_LOOKUP_PAGE}`);
+                        page = await nextPage.all(path, cursor);
+                    }
+                    const removed = [];
+                    const moved = [];
+                    for (const old of page){
+                        const index = Number(old.chunk);
+                        const chunk = chunks[index];
+                        if (!chunk || old.content_identity !== identified[index].identity) {
+                            removed.push([
+                                path,
+                                index
+                            ]);
+                        } else {
+                            kept.add(index);
+                            if (Number(old.start_line) !== chunk.startLine || Number(old.end_line) !== chunk.endLine) moved.push([
+                                chunk.startLine,
+                                chunk.endLine,
+                                path,
+                                index
+                            ]);
+                        }
+                    }
+                    if (removed.length > 0) await db.runBatch('DELETE FROM embeddings WHERE "path" = ? AND chunk = ?', removed);
+                    if (moved.length > 0) await db.runBatch('UPDATE embeddings SET start_line = ?, end_line = ? WHERE "path" = ? AND chunk = ?', moved);
+                    if (page.length < CHUNK_LOOKUP_PAGE) break;
+                    cursor = Number(page[page.length - 1].chunk);
+                }
+            }
+            const before = rows.length;
+            chunks.forEach((chunk, index)=>{
+                if (!kept.has(index)) rows.push([
                     path,
-                    idx,
-                    c.startLine,
-                    c.endLine
-                ]));
-            texts.set(path, chunks.map((c)=>c.text));
+                    index,
+                    chunk.startLine,
+                    chunk.endLine,
+                    identified[index].identity
+                ]);
+            });
+            // No new pending rows means no retained handoff. Matching pending rows can use
+            // embedPending's exact indexed-source fallback, including after repeated store calls.
+            if (rows.length > before) texts.set(path, identified);
         }
-        if (rows.length === 0) return;
         // scale and vector are written by the embed pass, not here, so they are not in the column list
         // and take the table's own default on the append path.
         await appendRows(db, 'embeddings', [
             'path',
             'chunk',
             'start_line',
-            'end_line'
-        ], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, scale, vector) VALUES (?, ?, ?, ?, NULL, NULL)', rows);
-        // The text extract() just computed, handed to the embed pass that follows in this same
-        // process rather than re-derived from the file there (src/embed/handoff.ts).
+            'end_line',
+            'content_identity'
+        ], 'INSERT INTO embeddings ("path", chunk, start_line, end_line, content_identity, scale, vector) VALUES (?, ?, ?, ?, ?, NULL, NULL)', rows);
+        // Hand off only paths with new pending rows; missing entries use the indexed source.
         stashChunkText(db, texts);
     },
     enabledForFile (_cfg, file) {
@@ -81,6 +140,8 @@ export const embed = {
         // W3b: chunkTokens rides the version token, since it's the one owner lever the chunker
         // itself takes -- changing or clearing it must rebuild exactly like a chunker version bump.
         const chunkVersion = e.chunkTokens !== undefined ? `${CHUNK_VERSION}:${e.chunkTokens}` : CHUNK_VERSION;
-        return `embed:${e.provider}:${e.model}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;
+        const endpoint = embedEndpointIdentity(e.provider, e.url);
+        const endpointPart = endpoint === undefined ? '' : `:endpoint=${endpoint}`;
+        return `embed:${e.provider}:${e.model}${endpointPart}:${chunkVersion}${identity !== undefined ? `@${identity}` : ''}`;
     }
 };