sensemaking 0.25.1 → 0.27.0

package/dist/cjs/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = (await store.vectors.pending()).filter((row) => allowed?.has(row.path) ?? true);\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map<string, string[]>();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && chunkIdxs.every((idx) => idx < texts.length)) {\n      for (const idx of chunkIdxs) jobs.push({ path, chunk: idx, text: texts[idx] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const idx of chunkIdxs) {\n      const chunk = chunks[idx];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      jobs.push({ path, chunk: idx, text: chunk.text });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["embedPending","ensureDocumentEmbeddings","hasEmbedding","semanticCandidates","similarNotes","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","store","cfg","allowed","takeChunkText","provider","dirty","storeDims","byPath","row","list","textByPath","sourceStmt","jobs","report","i","batch","vectors","slice","batchCap","embedDocuments","map","j","text","writeVectors","job","q","Int8Array","round","path","chunk","vector","Buffer","from","buffer","tick","min","length","getProvider","pending","filter","has","STORE_DIMS","Map","get","push","set","prepare","chunkIdxs","texts","idx","source","chunks","every","SenseError","parseSource","relPath","absPath","mtimeMs","ctimeMs","size","byteLength","presets","embed","doc","extracted","checkLanguageFit","progress","finish","_cfg","terms","fetch","rootDir","qv","baseDir","match","t","includes","join","embedQuery","candidates","hasVector","opts","similar"],"mappings":";;;;;;;;;;;QA6BsBA;eAAAA;;QA8DAC;eAAAA;;QAsBAC;eAAAA;;QAbAC;eAAAA;;QAmBAC;eAAAA;;QAzGNC;eAAAA;;;wBAbW;uBAEL;0BACG;uBACG;yBAEE;yBACG;0BACL;uBACD;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAIpB,SAASA,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAKO,SAAehB,aAAakB,KAAY,EAAEC,GAAW,EAAEC,OAA6B;;mBAatEC,gBAZbC,UACAC,OAEAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,YACAC,YAEAC,MACD,4BAAA,oBAAA,yBAAA,YAAA,aA4BCC,QACGC;;;;;4BACDC,OACAC;;;;oCADAD,QAAQH,KAAKK,KAAK,CAACH,GAAGA,IAAIV,SAASc,QAAQ;oCACjC;;wCAAMd,SAASe,cAAc,CAACJ,MAAMK,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;;;;oCAA/DN,UAAU;oCAChB;;wCAAMhB,MAAMgB,OAAO,CAACO,YAAY,CAC9BR,MAAMK,GAAG,CAAC,SAACI,KAAKH;4CACd,IAAqBlC,WAAAA,QAAQ6B,OAAO,CAACK,EAAE,EAAEf,WAAW,OAA5Cf,IAAaJ,SAAbI,GAAGM,QAAUV,SAAVU;4CACX,IAAM4B,IAAI,IAAIC,UAAUpB;4CACxB,IAAK,IAAIZ,IAAI,GAAGA,IAAIY,WAAWZ,IAAK+B,CAAC,CAAC/B,EAAE,GAAGC,KAAKgC,KAAK,CAACpC,CAAC,CAACG,EAAE,GAAGG;4CAC7D,OAAO;gDAAE+B,MAAMJ,IAAII,IAAI;gDAAEC,OAAOL,IAAIK,KAAK;gDAAEhC,OAAAA;gDAAOiC,QAAQC,OAAOC,IAAI,CAACP,EAAEQ,MAAM;4CAAE;wCAClF;;;oCANF;oCAQApB,OAAOqB,IAAI,CAACvC,KAAKwC,GAAG,CAACrB,IAAIV,SAASc,QAAQ,EAAEN,KAAKwB,MAAM;;;;;;oBACzD;oBAzDiB;;wBAAMC,IAAAA,uBAAW,EAACpC;;;oBAA7BG,WAAW;oBACF;;wBAAMJ,MAAMgB,OAAO,CAACsB,OAAO;;;oBAApCjC,QAAQ,AAAC,cAA+BkC,MAAM,CAAC,SAAC/B;;uCAAQN,oBAAAA,8BAAAA,QAASsC,GAAG,CAAChC,IAAIoB,IAAI,wCAAK;;oBACxF,IAAIvB,MAAM+B,MAAM,KAAK,GAAG;;;oBAClB9B,YAAYX,KAAKwC,GAAG,CAACM,mBAAU,EAAErC,SAASf,IAAI;oBAE9CkB,SAAS,IAAImC;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAarC,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdG,MAAN;;4BACGC,QAAOF,cAAAA,OAAOoC,GAAG,CAACnC,IAAIoB,IAAI,eAAnBrB,yBAAAA;4BACbE,KAAKmC,IAAI,CAACpC,IAAIqB,KAAK;4BACnBtB,OAAOsC,GAAG,CAACrC,IAAIoB,IAAI,EAAEnB;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC,cAAaP,iBAAAA,IAAAA,wBAAa,EAACH,oBAAdG,4BAAAA,iBAAwB,IAAIuC;oBAC5B;;wBAAM1C,MAAM8C,OAAO,CAAC;;;oBAAjCnC,aAAa;oBAEbC;oBACD,mCAAA,4BAAA;;;;;;;;;;yCAAOgB,MAAMmB,WACVC,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KAKPC,QAEAC,QACD,4BAAA,oBAAA,iBAAA,YAAA,QAAMF,MACHpB;;;;mEAZL,kBAAOD,uBAAMmB;oCACVC,QAAQtC,WAAWiC,GAAG,CAACf;oCAC7B,IAAIoB,SAASD,UAAUK,KAAK,CAAC,SAACH;+CAAQA,MAAMD,MAAMZ,MAAM;wCAAG;wCACpD,kCAAA,2BAAA;;4CAAL,IAAK,YAAaW,gCAAb,6BAAA,QAAA,yBAAA;gDAAME,MAAN;gDAAwBrC,KAAKgC,IAAI,CAAC;oDAAEhB,MAAAA;oDAAMC,OAAOoB;oDAAK3B,MAAM0B,KAAK,CAACC,IAAI;gDAAC;;;4CAAvE;4CAAA;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;wCACL;;4CAAA;;oCACF;oCAGgB;;wCAAMtC,WAAWgC,GAAG,CAACf;;;oCAA/BsB,SAAU;oCAChB,IAAI,CAACA,QAAQ,MAAM,IAAIG,oBAAU,CAAC,mBAAmB,AAAC,sCAA0C,OAALzB,MAAK;oCAC1FuB,SAASG,IAAAA,oBAAW,EAAC;wCAAEC,SAAS3B;wCAAM4B,SAAS5B;wCAAM6B,SAAS;wCAAGC,SAAS;wCAAGC,MAAM5B,OAAO6B,UAAU,CAACV,OAAO5B,IAAI;wCAAGuC,OAAO;wCAAMC,OAAO;oCAAK,GAAGZ,OAAO5B,IAAI;wCAAGwC,cAAK;uCAAG7D,KAAK8D,GAAG,CAACC,SAAS,CAACF,KAAK;oCAC9L,mCAAA,4BAAA;;wCAAL,IAAK,aAAaf,gCAAb,8BAAA,SAAA,0BAAA,kCAAwB;4CAAlBE,OAAN;4CACGpB,QAAQsB,MAAM,CAACF,KAAI;4CACzB,IAAI,CAACpB,OAAO,MAAM,IAAIwB,oBAAU,CAAC,mBAAmB,AAAC,+BAAmC,OAALzB,MAAK;4CACxFhB,KAAKgC,IAAI,CAAC;gDAAEhB,MAAAA;gDAAMC,OAAOoB;gDAAK3B,MAAMO,MAAMP,IAAI;4CAAC;wCACjD;;wCAJK;wCAAA;;;iDAAA,8BAAA;gDAAA;;;gDAAA;sDAAA;;;;;;;;;oBAKP;oBAhBK,aAA2Bf;;;2BAA3B,8BAAA,SAAA;;;;;;;;;;;;oBAAA;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAkBL,sFAAsF;oBACtF,2EAA2E;oBAC3E;;wBAAM0D,IAAAA,2BAAgB,EACpBjE,OACAI,UACAQ,KAAKQ,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;oBAMA,0FAA0F;oBAC1F,4BAA4B;oBACtBT,SAASqD,IAAAA,oBAAQ,EAAC,oBAAoBtD,KAAKwB,MAAM;oBAC9CtB,IAAI;;;yBAAGA,CAAAA,IAAIF,KAAKwB,MAAM,AAAD;;;;;;;;;;;;oBAAGtB,KAAKV,SAASc,QAAQ;;;;;;oBAavDL,OAAOsD,MAAM;;;;;;IACf;;AAEO,SAAepF,yBAAyBiB,KAAY,EAAEoE,IAAY,EAAElE,OAA4B;;YAC/FoC;;;;oBAAW;;wBAAMtC,MAAMgB,OAAO,CAACsB,OAAO;;;oBAAtCA,UAAU,AAAC,cAA+BC,MAAM,CAAC,SAAC/B;+BAAQN,QAAQsC,GAAG,CAAChC,IAAIoB,IAAI;;oBACpF,IAAIU,QAAQF,MAAM,GAAG,GAAG;wBACtB,MAAM,IAAIiB,oBAAU,CAAC,mBAAmB,AAAC,GAAiB,OAAff,QAAQF,MAAM,EAAC;oBAC5D;;;;;;IACF;;AAIO,SAAenD,mBAAmBe,KAAY,EAAEC,GAAW,EAAEoE,KAAa,EAAEC,KAAa,EAAEpE,OAAqB;;YACrG,cAIFmE,cAJRE,SAEAnE,UACAE,WACAgB,MACYnC,UAAPqF;;;;oBALLD,WAAU,eAAA,AAACtE,IAAgCsE,OAAO,cAAxC,0BAAA,eAA4C,AAACtE,IAAgCwE,OAAO;oBACpG,IAAI,CAACF,SAAS,MAAM,IAAIlB,oBAAU,CAAC,eAAe;oBACjC;;wBAAMhB,IAAAA,uBAAW,EAACpC;;;oBAA7BG,WAAW;oBACXE,YAAYX,KAAKwC,GAAG,CAACM,mBAAU,EAAErC,SAASf,IAAI;oBAC9CiC,OAAO,EAAC+C,eAAAA,MAAMK,KAAK,CAAC,2naAAZL,0BAAAA,mBAAsC9B,MAAM,CAAC,SAACoC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIE,IAAI,CAAC;oBACxF;;wBAAMzE,SAAS0E,UAAU,CAACxD;;;oBAAlCnC,WAAAA;wBAAQ;wBAAiCmB;wBAAW;wBAA3DkE,KAAOrF,SAAVI;oBAER;;wBAAOS,MAAMgB,OAAO,CAAC+D,UAAU,CAACP,IAAIlE,WAAWgE,OAAOpE;;;;IACxD;;AAIO,SAAelB,aAAagB,KAAY,EAAE4B,IAAY;;;YAC3D;;gBAAO5B,MAAMgB,OAAO,CAACgE,SAAS,CAACpD;;;IACjC;;AAIO,SAAe1C,aAAac,KAAY,EAAEoE,IAAY,EAAExC,IAAY,EAAEqD,IAAgE;;;YAC3I;;gBAAOjF,MAAMgB,OAAO,CAACkE,OAAO,CAACtD,MAAMqD;;;IACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["embedPending","ensureDocumentEmbeddings","hasEmbedding","prepareSemanticQuery","semanticCandidates","similarNotes","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","SenseError","store","cfg","allowed","expectedSignature","options","takeChunkText","expectedDurableSignature","pendingStmt","dirty","provider","submittedSignature","storeDims","byPath","row","list","textByPath","sourceStmt","jobs","report","i","batch","vectors","writes","slice","batchCap","signal","throwIfAborted","featureSignature","FEATURES","embedDocuments","map","j","text","job","q","Int8Array","round","path","chunk","vector","Buffer","from","buffer","transaction","currentStmt","current","identities","writeVectors","getMeta","FEATURE_SIGNATURE_META_KEY","prepare","join","all","flatMap","Map","Number","content_identity","some","get","identity","tick","min","length","undefined","embeddingSignatureCompatible","filter","has","getProvider","STORE_DIMS","push","set","pending","texts","source","chunks","identified","every","parseSource","relPath","absPath","mtimeMs","ctimeMs","size","byteLength","presets","embed","doc","extracted","identifyChunkText","checkLanguageFit","progress","finish","_cfg","terms","rootDir","queryVector","qv","baseDir","match","t","includes","embedQuery","prepared","fetch","candidates","hasVector","opts","similar"],"mappings":";;;;;;;;;;;QAuCsBA;eAAAA;;QAqGAC;eAAAA;;QAoCAC;eAAAA;;QAtBAC;eAAAA;;QAgBNC;eAAAA;;QAYMC;eAAAA;;QApKNC;eAAAA;;;uBAlBmD;wBACxC;uBAEL;wBACG;0BACA;wBACG;2BACe;wBACnB;2BACqB;yBAEI;yBAChB;0BACL;uBACsB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAI3C,SAASA,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,IAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,KAAI,GAAGA,KAAIL,MAAMK,KAAKH,CAAC,CAACG,GAAE,GAAGN,IAAI,CAACM,GAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC,GAAAA;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,KAAI,GAAGA,KAAIL,MAAMK,KAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,GAAE;IAChE,OAAO;QAAEH,GAAAA;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAIC,oBAAU,CAAC,mBAAmB,AAAC,GAAS,OAAPD,QAAO;AACrD;AAIO,SAAepB;wCAAasB,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC;mBAAEC,SA4B3GC,gBA3BnBD,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;QACtF,2EAA2E;QAC3EA,kBAMAA,kBA3DME,gCAMAC,aACAC,OAIAC,UAEAC,oBAIAC,WAEAC,QACD,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,KACID,aAAPE,MAKFC,YACAC,YAEAC,MACD,4BAAA,oBAAA,yBAAA,YAAA,aAiCCC,QACGC;;;;;;4BAEPf,iBAMAA,kBAQAA,kBAfMgB,OAGAC,SAMAC;;;;oCATAF,QAAQH,KAAKM,KAAK,CAACJ,GAAGA,IAAIV,SAASe,QAAQ;qCACjDpB,kBAAAA,QAAQqB,MAAM,cAAdrB,sCAAAA,gBAAgBsB,cAAc;oCAC9B,IAAIC,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ,MAAMlB,oBAAoB,MAAMb,mBAAmB;oCACrE;;wCAAMY,SAASoB,cAAc,CAC3CT,MAAMU,GAAG,CAAC,SAACC;mDAAMA,EAAEC,IAAI;4CACvB5B;;;oCAFIiB,UAAU;qCAIhBjB,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oCAC9B,IAAIC,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ,MAAMlB,oBAAoB,MAAMb,mBAAmB;oCAC/EyB,SAASF,MAAMU,GAAG,CAAC,SAACG,KAAKF;wCAC7B,IAAqB/C,WAAAA,QAAQqC,OAAO,CAACU,EAAE,EAAEpB,WAAW,OAA5CvB,IAAaJ,SAAbI,GAAGM,QAAUV,SAAVU;wCACX,IAAMwC,IAAI,IAAIC,UAAUxB;wCACxB,IAAK,IAAIpB,IAAI,GAAGA,IAAIoB,WAAWpB,IAAK2C,CAAC,CAAC3C,EAAE,GAAGC,KAAK4C,KAAK,CAAChD,CAAC,CAACG,EAAE,GAAGG;wCAC7D,OAAO;4CAAE2C,MAAMJ,IAAII,IAAI;4CAAEC,OAAOL,IAAIK,KAAK;4CAAE5C,OAAAA;4CAAO6C,QAAQC,OAAOC,IAAI,CAACP,EAAEQ,MAAM;wCAAE;oCAClF;qCACAtC,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oCAC9B;;wCAAM1B,MAAM2C,WAAW,CAAC;;oDAQCC,cAPvB,wFAAwF;gDACxF,mFAAmF;gDACnFxC,iBAEAA,kBAMAA,kBAJMwC,aACAC,SACAC;;;;6DANN1C,kBAAAA,QAAQqB,MAAM,cAAdrB,sCAAAA,gBAAgBsB,cAAc;4DAC9B;;gEAAM1B,MAAMqB,OAAO,CAAC0B,YAAY,CAACzB;;;4DAAjC;6DACAlB,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;4DACzB;;gEAAMsB,IAAAA,iBAAO,EAAChD,OAAOiD,uCAA0B;;;4DAApD,IAAI,AAAC,kBAAsD3C,0BAA0B,MAAMT,mBAAmB;4DAC1F;;gEAAMG,MAAMkD,OAAO,CAAC,AAAC,gEAA0H,OAA3D9B,MAAMU,GAAG,CAAC;2EAAM;mEAA8BqB,IAAI,CAAC;;;4DAArJP,cAAc;4DACH;;gEAAMA,CAAAA,eAAAA,aAAYQ,GAAG,OAAfR,cAAgB,qBAAGxB,MAAMiC,OAAO,CAAC,SAACpB;2EAAQ;wEAACA,IAAII,IAAI;wEAAEJ,IAAIK,KAAK;qEAAC;;;;4DAAhFO,UAAW;4DACXC,aAAa,IAAIQ,IAAIT,QAAQf,GAAG,CAAC,SAACjB;uEAAQ;oEAAE,GAAe0C,OAAb1C,IAAIwB,IAAI,EAAC,MAAsB,OAAlBkB,OAAO1C,IAAIyB,KAAK;oEAAKzB,IAAI2C,gBAAgB;iEAAC;;4DAC3G,IAAIpC,MAAMqC,IAAI,CAAC,SAACxB;uEAAQa,WAAWY,GAAG,CAAC,AAAC,GAAezB,OAAbA,IAAII,IAAI,EAAC,MAAc,OAAVJ,IAAIK,KAAK,OAAQL,IAAI0B,QAAQ;gEAAG,MAAM9D,mBAAmB;6DAChHO,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;;;;;;4CAChC;;;;oCAZA;oCAaAR,OAAO0C,IAAI,CAACpE,KAAKqE,GAAG,CAAC1C,IAAIV,SAASe,QAAQ,EAAEP,KAAK6C,MAAM;;;;;;oBACzD;oBAjG8H1D,UAAAA,oEAA4B,CAAC;qBAC3JA,kBAAAA,QAAQqB,MAAM,cAAdrB,sCAAAA,gBAAgBsB,cAAc;yBACGvB,CAAAA,sBAAsB4D,SAAQ,GAA9B5D;;;;oBAAkC;;wBAAM6C,IAAAA,iBAAO,EAAChD,OAAOiD,uCAA0B;;;2BAA/C;;;;;;2BAAmD9C;;;oBAAhHG;qBACNF,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBAC9B,IAAI,CAACsC,IAAAA,yCAA4B,EAAC1D,0BAA0BqB,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ,IAAI;wBAC5F,MAAM/B,mBAAmB;oBAC3B;oBAEoB;;wBAAMG,MAAMkD,OAAO,CAAC;;;oBAAlC3C,cAAc;oBACJ;;wBAAMA,YAAY6C,GAAG;;;oBAA/B5C,QAAQ,AAAE,cAA8FyD,MAAM,CAAC,SAACpD;;uCAAQX,oBAAAA,8BAAAA,QAASgE,GAAG,CAACrD,IAAIwB,IAAI,wCAAK;;qBACxJjC,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBAC9B,IAAIlB,MAAMsD,MAAM,KAAK,GAAG;;;oBAEP;;wBAAMK,IAAAA,uBAAW,EAAClE,KAAKG;;;oBAAlCK,WAAW;qBACjBL,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBACxBhB,qBAAqBiB,IAAAA,yBAAgB,EAAC1B,KAAK2B,kBAAQ;oBACzD,IAAI,CAACoC,IAAAA,yCAA4B,EAAC1D,0BAA0BI,qBAAqB;wBAC/E,MAAMb,mBAAmB;oBAC3B;oBACMc,YAAYnB,KAAKqE,GAAG,CAACO,mBAAU,EAAE3D,SAASvB,IAAI;oBAE9C0B,SAAS,IAAI0C;oBACd,kCAAA,2BAAA;;wBAAL,IAAK,YAAa9C,4BAAb,6BAAA,QAAA,yBAAA,iCAAoB;4BAAdK,MAAN;;4BACGC,QAAOF,cAAAA,OAAO8C,GAAG,CAAC7C,IAAIwB,IAAI,eAAnBzB,yBAAAA;4BACbE,KAAKuD,IAAI,CAACxD;4BACVD,OAAO0D,GAAG,CAACzD,IAAIwB,IAAI,EAAEvB;wBACvB;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;oBAMCC,cAAaV,iBAAAA,IAAAA,wBAAa,EAACL,oBAAdK,4BAAAA,iBAAwB,IAAIiD;oBAC5B;;wBAAMtD,MAAMkD,OAAO,CAAC;;;oBAAjClC,aAAa;oBAEbC;oBACD,mCAAA,4BAAA;;;;;;;;;;yCAAOoB,MAAMkC,SAShBnE,iBARMoE,OAEC,2BAAA,mBAAA,gBAAA,WAAA,OAAM3D,KAKP4D,QAGAC,QACD,4BAAA,oBAAA,iBAAA,YAAA,QAAM7D,MACHyB,OAEAqC;;;;mEAfL,kBAAOtC,uBAAMkC;oCACVC,QAAQzD,WAAW2C,GAAG,CAACrB;oCAC7B,IAAImC,SAASD,QAAQK,KAAK,CAAC,SAAC/D;4CAAQ2D;+CAAAA,EAAAA,mBAAAA,KAAK,CAAC3D,IAAIyB,KAAK,CAAC,cAAhBkC,uCAAAA,iBAAkBb,QAAQ,MAAK9C,IAAI2C,gBAAgB;wCAAG;wCACnF,kCAAA,2BAAA;;4CAAL,IAAK,YAAae,8BAAb,6BAAA,QAAA,yBAAA;gDAAM1D,MAAN;gDAAsBI,KAAKoD,IAAI,CAAC;oDAAEhC,MAAAA;oDAAMC,OAAOzB,IAAIyB,KAAK;mDAAKkC,KAAK,CAAC3D,IAAIyB,KAAK,CAAC;;;4CAA7E;4CAAA;;;qDAAA,6BAAA;oDAAA;;;oDAAA;0DAAA;;;;wCACL;;4CAAA;;oCACF;oCAGgB;;wCAAMtB,WAAW0C,GAAG,CAACrB;;;oCAA/BoC,SAAU;qCAChBrE,kBAAAA,QAAQqB,MAAM,cAAdrB,sCAAAA,gBAAgBsB,cAAc;oCAC9B,IAAI,CAAC+C,QAAQ,MAAM,IAAI1E,oBAAU,CAAC,mBAAmB,AAAC,sCAA0C,OAALsC,MAAK;oCAC1FqC,SAASG,IAAAA,qBAAW,EAAC;wCAAEC,SAASzC;wCAAM0C,SAAS1C;wCAAM2C,SAAS;wCAAGC,SAAS;wCAAGC,MAAM1C,OAAO2C,UAAU,CAACV,OAAOzC,IAAI;wCAAGoD,OAAO;wCAAMC,OAAO;oCAAK,GAAGZ,OAAOzC,IAAI;wCAAGqD,cAAK;uCAAGpF,KAAKqF,GAAG,CAACC,SAAS,CAACF,KAAK;oCAC9L,mCAAA,4BAAA;;wCAAL,IAAK,aAAad,8BAAb,8BAAA,SAAA,0BAAA,kCAAsB;4CAAhB1D,OAAN;4CACGyB,QAAQoC,MAAM,CAAC7D,KAAIyB,KAAK,CAAC;4CAC/B,IAAI,CAACA,OAAO,MAAM,IAAIvC,oBAAU,CAAC,mBAAmB,AAAC,+BAAmC,OAALsC,MAAK;4CAClFsC,aAAaa,IAAAA,4BAAiB,EAAClD,MAAMN,IAAI;4CAC/C,IAAI2C,WAAWhB,QAAQ,KAAK9C,KAAI2C,gBAAgB,EAAE,MAAM3D,mBAAmB,AAAC,iBAAwBgB,OAARwB,MAAK,KAAa,OAAVxB,KAAIyB,KAAK,EAAC;4CAC9GrB,KAAKoD,IAAI,CAAC;gDAAEhC,MAAAA;gDAAMC,OAAOzB,KAAIyB,KAAK;+CAAKqC;wCACzC;;wCANK;wCAAA;;;iDAAA,8BAAA;gDAAA;;;gDAAA;sDAAA;;;;;;;;;oBAOP;oBAnBK,aAAyB/D;;;2BAAzB,8BAAA,SAAA;;;;;;;;;;;;oBAAA;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,8BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;qBAuBLR,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBAC9B;;wBAAM+D,IAAAA,2BAAgB,EACpBzF,OACAS,UACAQ,KAAKa,GAAG,CAAC,SAACC;mCAAMA,EAAEC,IAAI;;;;oBAHxB;qBAKA5B,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBAE9B,0FAA0F;oBAC1F,4BAA4B;oBACtBR,SAASwE,IAAAA,oBAAQ,EAAC,oBAAoBzE,KAAK6C,MAAM;oBAC9C3C,IAAI;;;yBAAGA,CAAAA,IAAIF,KAAK6C,MAAM,AAAD;;;;;;;;;;;;oBAAG3C,KAAKV,SAASe,QAAQ;;;;;;oBAgCvDN,OAAOyE,MAAM;;;;;;IACf;;AAEO,SAAehH,yBAAyBqB,KAAY,EAAE4F,IAAY,EAAE1F,OAA4B;;YAC/FqE;;;;oBAAW;;wBAAMvE,MAAMqB,OAAO,CAACkD,OAAO;;;oBAAtCA,UAAU,AAAC,cAA+BN,MAAM,CAAC,SAACpD;+BAAQX,QAAQgE,GAAG,CAACrD,IAAIwB,IAAI;;oBACpF,IAAIkC,QAAQT,MAAM,GAAG,GAAG;wBACtB,MAAM,IAAI/D,oBAAU,CAAC,mBAAmB,AAAC,GAAiB,OAAfwE,QAAQT,MAAM,EAAC;oBAC5D;;;;;;IACF;;AASO,SAAejF;wCAAqBoB,GAAW,EAAE4F,KAAa;YAAEzF,SACrD,cAEFyF,cACdzF,iBAEAA,kBAGAA,kBARM0F,SAEA9D,MAEAvB,UAEAE,WACAoF,aAEY/G,UAAPgH;;;;;oBAV0D5F,UAAAA,oEAA4B,CAAC;oBAC5F0F,WAAU,eAAA,AAAC7F,IAAgC6F,OAAO,cAAxC,0BAAA,eAA4C,AAAC7F,IAAgCgG,OAAO;oBACpG,IAAI,CAACH,SAAS,MAAM,IAAI/F,oBAAU,CAAC,eAAe;oBAC5CiC,OAAO,EAAC6D,eAAAA,MAAMK,KAAK,CAAC,2naAAZL,0BAAAA,mBAAsC5B,MAAM,CAAC,SAACkC;+BAAM,CAAC;4BAAC;4BAAO;4BAAM;4BAAO;yBAAO,CAACC,QAAQ,CAACD;uBAAIhD,IAAI,CAAC;qBAClH/C,kBAAAA,QAAQqB,MAAM,cAAdrB,sCAAAA,gBAAgBsB,cAAc;oBACb;;wBAAMyC,IAAAA,uBAAW,EAAClE,KAAKG;;;oBAAlCK,WAAW;qBACjBL,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBACxBf,YAAYnB,KAAKqE,GAAG,CAACO,mBAAU,EAAE3D,SAASvB,IAAI;oBAChC;;wBAAMuB,SAAS4F,UAAU,CAACrE,MAAM5B;;;oBAA9C2F,cAAc;qBACpB3F,mBAAAA,QAAQqB,MAAM,cAAdrB,uCAAAA,iBAAgBsB,cAAc;oBACZ1C,WAAAA,QAAQ+G,aAAapF,WAAW,QAAvCqF,KAAOhH,SAAVI;oBACR;;wBAAO;4BAAEmD,QAAQyD;4BAAIrF,WAAAA;wBAAU;;;;IACjC;;AAIO,SAAS7B,mBAAmBkB,KAAY,EAAEsG,QAA+B,EAAEC,KAAa,EAAErG,OAAqB;IACpH,OAAOF,MAAMqB,OAAO,CAACmF,UAAU,CAACF,SAAS/D,MAAM,EAAE+D,SAAS3F,SAAS,EAAE4F,OAAOrG;AAC9E;AAIO,SAAetB,aAAaoB,KAAY,EAAEqC,IAAY;;;YAC3D;;gBAAOrC,MAAMqB,OAAO,CAACoF,SAAS,CAACpE;;;IACjC;;AAIO,SAAetD,aAAaiB,KAAY,EAAE4F,IAAY,EAAEvD,IAAY,EAAEqE,IAAgE;;;YAC3I;;gBAAO1G,MAAMqB,OAAO,CAACsF,OAAO,CAACtE,MAAMqE;;;IACrC"}

package/dist/cjs/embed/registry.d.cts

@@ -1,3 +1,3 @@
 import type { Config } from '../config/index.js';
-import type { EmbedProvider } from './types.js';
-export declare function getProvider(cfg: Config): Promise<EmbedProvider>;
+import type { EmbedCallOptions, EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config, options?: EmbedCallOptions): Promise<EmbedProvider>;

package/dist/cjs/embed/registry.d.ts

@@ -1,3 +1,3 @@
 import type { Config } from '../config/index.js';
-import type { EmbedProvider } from './types.js';
-export declare function getProvider(cfg: Config): Promise<EmbedProvider>;
+import type { EmbedCallOptions, EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config, options?: EmbedCallOptions): Promise<EmbedProvider>;

package/dist/cjs/embed/registry.js

@@ -11,6 +11,7 @@ Object.defineProperty(exports, "getProvider", {
 var _indexts = require("../config/index.js");
 var _errorsts = require("../errors.js");
 var _coherets = require("./cohere.js");
+var _endpointts = require("./endpoint.js");
 var _openaits = require("./openai.js");
 var _staticts = require("./static.js");
 function _define_property(obj, key, value) {
@@ -76,24 +77,55 @@ var REGISTRY = {
     }
 };
 var providers = new Map();
+function aborted(reason) {
+    return Promise.reject(reason);
+}
+function waitForProvider(provider, signal) {
+    if (signal.aborted) return aborted(signal.reason);
+    return new Promise(function(resolve, reject) {
+        var cleanup = function cleanup() {
+            return signal.removeEventListener('abort', onAbort);
+        };
+        var onAbort = function onAbort() {
+            cleanup();
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', onAbort, {
+            once: true
+        });
+        provider.then(function(value) {
+            cleanup();
+            resolve(value);
+        }, function(err) {
+            cleanup();
+            reject(err);
+        });
+        if (signal.aborted) onAbort();
+    });
+}
 function getProvider(cfg) {
-    var _e_url, _e_key, _e_languages;
+    var options = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : {};
+    var _e_key, _e_languages;
+    var _options_signal;
+    // A caller that has already left must not start shared model initialization.
+    if ((_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.aborted) return aborted(options.signal.reason);
     var e = (0, _indexts.embedConfig)(cfg);
     if (!e) throw new _errorsts.SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');
     var factory = REGISTRY[e.provider];
     if (!factory) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "'.concat(e.provider, '" has no provider implementation in this build'));
+    var endpoint = (0, _endpointts.effectiveEmbedEndpoint)(e.provider, e.url);
     // Every input that shapes the provider is in the key: two trees sharing an endpoint and
     // model but declaring different languages or key env vars are two providers, not one.
     var sig = JSON.stringify([
         e.provider,
         e.model,
-        (_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '',
+        endpoint !== null && endpoint !== void 0 ? endpoint : '',
         (_e_key = e.key) !== null && _e_key !== void 0 ? _e_key : '',
         (_e_languages = e.languages) !== null && _e_languages !== void 0 ? _e_languages : []
     ]);
     var p = providers.get(sig);
     if (!p) {
-        p = factory(e.model, e.url, e.key);
+        p = factory(e.model, endpoint, e.key);
         // Owner-declared languages override a provider's own: the config is the one home for
         // what no model card can state (http models have no fetchable card here).
         if (e.languages) p = p.then(function(prov) {
@@ -108,6 +140,6 @@ function getProvider(cfg) {
             return providers.delete(sig);
         });
     }
-    return p;
+    return options.signal ? waitForProvider(p, options.signal) : p;
 }
 /* CJS INTEROP */ if (exports.__esModule && exports.default) { try { Object.defineProperty(exports.default, '__esModule', { value: true }); for (var key in exports) { exports.default[key] = exports[key]; } } catch (_) {}; module.exports = exports.default; }

package/dist/cjs/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["getProvider","REGISTRY","static","model","staticProvider","openai","url","key","openaiProvider","cohere","cohereProvider","providers","Map","cfg","e","embedConfig","SenseError","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":";;;;+BAoBgBA;;;eAAAA;;;uBAnBY;wBACD;wBACI;wBACA;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK/B,yFAAyF;AACzF,iCAAiC;AACjC,IAAMC,WAA4C;IAChDC,QAAQ,SAARA,QAASC;eAAUC,IAAAA,wBAAc,EAACD;;IAClCE,QAAQ,SAARA,OAASF,OAAOG,KAAKC;eAAQC,IAAAA,wBAAc,EAACL,OAAOG,KAAKC;;IACxDE,QAAQ,SAARA,OAASN,OAAOG,KAAKC;eAAQG,IAAAA,wBAAc,EAACP,OAAOG,KAAKC;;AAC1D;AAEA,IAAMI,YAAY,IAAIC;AAEf,SAASZ,YAAYa,GAAW;QAOYC,QAAaA,QAAaA;IAN3E,IAAMA,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIE,oBAAU,CAAC,kBAAkB;IAC/C,IAAMC,UAAUhB,QAAQ,CAACa,EAAEI,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAID,oBAAU,CAAC,eAAe,AAAC,mBAA6B,OAAXF,EAAEI,QAAQ,EAAC;IAChF,wFAAwF;IACxF,sFAAsF;IACtF,IAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACP,EAAEI,QAAQ;QAAEJ,EAAEX,KAAK;SAAEW,SAAAA,EAAER,GAAG,cAALQ,oBAAAA,SAAS;SAAIA,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,eAAAA,EAAEQ,SAAS,cAAXR,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIS,IAAIZ,UAAUa,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQH,EAAEX,KAAK,EAAEW,EAAER,GAAG,EAAEQ,EAAEP,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIO,EAAEQ,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,SAACC;mBAAU,wCAAKA;gBAAMJ,WAAWR,EAAEQ,SAAS;;;QACxEX,UAAUgB,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC;mBAAMjB,UAAUkB,MAAM,CAACV;;IACjC;IACA,OAAOI;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction aborted(reason: unknown): Promise<never> {\n  return Promise.reject(reason);\n}\n\nfunction waitForProvider(provider: Promise<EmbedProvider>, signal: AbortSignal): Promise<EmbedProvider> {\n  if (signal.aborted) return aborted(signal.reason);\n  return new Promise((resolve, reject) => {\n    const cleanup = () => signal.removeEventListener('abort', onAbort);\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    provider.then(\n      (value) => {\n        cleanup();\n        resolve(value);\n      },\n      (err: unknown) => {\n        cleanup();\n        reject(err);\n      }\n    );\n    if (signal.aborted) onAbort();\n  });\n}\n\nexport function getProvider(cfg: Config, options: EmbedCallOptions = {}): Promise<EmbedProvider> {\n  // A caller that has already left must not start shared model initialization.\n  if (options.signal?.aborted) return aborted(options.signal.reason);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  const endpoint = effectiveEmbedEndpoint(e.provider, e.url);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, endpoint ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, endpoint, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return options.signal ? waitForProvider(p, options.signal) : p;\n}\n"],"names":["getProvider","REGISTRY","static","model","staticProvider","openai","url","key","openaiProvider","cohere","cohereProvider","providers","Map","aborted","reason","Promise","reject","waitForProvider","provider","signal","resolve","cleanup","removeEventListener","onAbort","addEventListener","once","then","value","err","cfg","options","e","embedConfig","SenseError","factory","endpoint","effectiveEmbedEndpoint","sig","JSON","stringify","languages","p","get","prov","set","catch","delete"],"mappings":";;;;+BAgDgBA;;;eAAAA;;;uBA/CY;wBACD;wBACI;0BACQ;wBACR;wBACA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAK/B,yFAAyF;AACzF,iCAAiC;AACjC,IAAMC,WAA4C;IAChDC,QAAQ,SAARA,QAASC;eAAUC,IAAAA,wBAAc,EAACD;;IAClCE,QAAQ,SAARA,OAASF,OAAOG,KAAKC;eAAQC,IAAAA,wBAAc,EAACL,OAAOG,KAAKC;;IACxDE,QAAQ,SAARA,OAASN,OAAOG,KAAKC;eAAQG,IAAAA,wBAAc,EAACP,OAAOG,KAAKC;;AAC1D;AAEA,IAAMI,YAAY,IAAIC;AAEtB,SAASC,QAAQC,MAAe;IAC9B,OAAOC,QAAQC,MAAM,CAACF;AACxB;AAEA,SAASG,gBAAgBC,QAAgC,EAAEC,MAAmB;IAC5E,IAAIA,OAAON,OAAO,EAAE,OAAOA,QAAQM,OAAOL,MAAM;IAChD,OAAO,IAAIC,QAAQ,SAACK,SAASJ;QAC3B,IAAMK,UAAU;mBAAMF,OAAOG,mBAAmB,CAAC,SAASC;;QAC1D,IAAMA,UAAU;YACdF;YACAL,OAAOG,OAAOL,MAAM;QACtB;QACAK,OAAOK,gBAAgB,CAAC,SAASD,SAAS;YAAEE,MAAM;QAAK;QACvDP,SAASQ,IAAI,CACX,SAACC;YACCN;YACAD,QAAQO;QACV,GACA,SAACC;YACCP;YACAL,OAAOY;QACT;QAEF,IAAIT,OAAON,OAAO,EAAEU;IACtB;AACF;AAEO,SAASvB,YAAY6B,GAAW;QAAEC,UAAAA,iEAA4B,CAAC;QAUHC,QAAaA;QAR1ED;IADJ,6EAA6E;IAC7E,KAAIA,kBAAAA,QAAQX,MAAM,cAAdW,sCAAAA,gBAAgBjB,OAAO,EAAE,OAAOA,QAAQiB,QAAQX,MAAM,CAACL,MAAM;IACjE,IAAMiB,IAAIC,IAAAA,oBAAW,EAACH;IACtB,IAAI,CAACE,GAAG,MAAM,IAAIE,oBAAU,CAAC,kBAAkB;IAC/C,IAAMC,UAAUjC,QAAQ,CAAC8B,EAAEb,QAAQ,CAAC;IACpC,IAAI,CAACgB,SAAS,MAAM,IAAID,oBAAU,CAAC,eAAe,AAAC,mBAA6B,OAAXF,EAAEb,QAAQ,EAAC;IAChF,IAAMiB,WAAWC,IAAAA,kCAAsB,EAACL,EAAEb,QAAQ,EAAEa,EAAEzB,GAAG;IACzD,wFAAwF;IACxF,sFAAsF;IACtF,IAAM+B,MAAMC,KAAKC,SAAS,CAAC;QAACR,EAAEb,QAAQ;QAAEa,EAAE5B,KAAK;QAAEgC,qBAAAA,sBAAAA,WAAY;SAAIJ,SAAAA,EAAExB,GAAG,cAALwB,oBAAAA,SAAS;SAAIA,eAAAA,EAAES,SAAS,cAAXT,0BAAAA,eAAe,EAAE;KAAC;IAChG,IAAIU,IAAI9B,UAAU+B,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIP,QAAQH,EAAE5B,KAAK,EAAEgC,UAAUJ,EAAExB,GAAG;QACpC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIwB,EAAES,SAAS,EAAEC,IAAIA,EAAEf,IAAI,CAAC,SAACiB;mBAAU,wCAAKA;gBAAMH,WAAWT,EAAES,SAAS;;;QACxE7B,UAAUiC,GAAG,CAACP,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEI,KAAK,CAAC;mBAAMlC,UAAUmC,MAAM,CAACT;;IACjC;IACA,OAAOP,QAAQX,MAAM,GAAGF,gBAAgBwB,GAAGX,QAAQX,MAAM,IAAIsB;AAC/D"}

package/dist/cjs/embed/static.js

@@ -309,9 +309,11 @@ function staticProvider(model, root) {
                             dims: dims,
                             batchCap: BATCH_CAP,
                             languages: languages,
-                            embedDocuments: function embedDocuments(texts) {
+                            embedDocuments: function embedDocuments(texts, options) {
                                 return _async_to_generator(function() {
+                                    var _options_signal;
                                     return _ts_generator(this, function(_state) {
+                                        options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
                                         return [
                                             2,
                                             texts.map(one)
@@ -319,9 +321,11 @@ function staticProvider(model, root) {
                                     });
                                 })();
                             },
-                            embedQuery: function embedQuery(text) {
+                            embedQuery: function embedQuery(text, options) {
                                 return _async_to_generator(function() {
+                                    var _options_signal;
                                     return _ts_generator(this, function(_state) {
+                                        options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
                                         return [
                                             2,
                                             one(text)

package/dist/cjs/embed/static.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return { id: `static:${model}`, dims, batchCap: BATCH_CAP, languages, embedDocuments: async (texts) => texts.map(one), embedQuery: async (text) => one(text) };\n}\n"],"names":["staticProvider","BATCH_CAP","_require","require","Module","createRequire","model","root","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","map","embedQuery"],"mappings":";;;;+BAesBA;;;eAAAA;;;sBAfmB;iEACtB;wBACE;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAInF,SAAeH,eAAeM,KAAa,EAAEC,IAAa;;kBA+BpBC,sBAA7BA,4BAAAA,uBA9BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eACEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,0FAA0F;YAC1F,0FAA0F;YAC1F,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAhDIQ,IAAAA,uBAAc,EAACnC,QAAfmC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACpC,OAAO,SAACM,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATN,OAAM,KAAgBqC,OAAb/B,MAAK,UAAa,OAAL+B;2BAASpC;;;oBADjH,qFAAqF;oBACrFE,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACxC,OAAOC;oBACpCG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAAC1C,OAAOC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCH,OAAM,iCAAiD+C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN9C,OAAM;oBAC/EW,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBACnEY,YAAcnB,SAAS,2BAAvBmB;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAsB/E,mEAAmE;oBACnE;;wBAAO;4BAAEzC,IAAI,AAAC,UAAe,OAANxB;4BAASY,MAAAA;4BAAMsD,UAAUvE;4BAAWS,WAAAA;4BAAW+D,gBAAgB,SAAhBA,eAAuBC;;;;;4CAAUA,MAAMC,GAAG,CAACnD;;;;;4BAAMoD,YAAY,SAAZA,WAAmBnD;;;;;4CAASD,IAAIC;;;;;wBAAM;;;;IAC/J"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/static.ts"],"sourcesContent":["import { existsSync, readFileSync } from 'node:fs';\nimport Module from 'node:module';\nimport { join } from 'node:path';\nimport { SenseError } from '../errors.ts';\nimport { downloadModel, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages } from './store.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// @huggingface/tokenizers is a 540K static import; a top-level import would put it on every\n// command's module graph, so it's deferred to a synchronous require inside staticProvider.\nconst _require = typeof require === 'undefined' ? Module.createRequire(import.meta.url) : require;\n\n// Model2Vec safetensors + pure-JS tokenizer. Encode convention from model2vec/model.py: no\n// special tokens, drop unk ids, mean-pool, L2-normalize.\nexport async function staticProvider(model: string, root?: string): Promise<EmbedProvider> {\n  let dir: string;\n  let languages: string[] | undefined;\n  if (isDownloadable(model)) {\n    // Naming a HF id is consent: fetch whatever is missing now, announcing it on stderr.\n    dir = await downloadModel(model, (file, into) => console.error(`sense: fetching ${model}/${file} into ${into}`), root);\n    // Cached beside the repo; a local model directory has no card to read, so it has none.\n    const cached = readLanguages(model, root);\n    languages = cached && cached.length > 0 ? cached : undefined;\n  } else {\n    dir = modelDir(model, root);\n    for (const file of MODEL_FILES) {\n      if (!existsSync(join(dir, file))) {\n        throw new SenseError('EMBED_MODEL_MISSING', `embed model ${model} is not available (looked in ${dir}); expected ${MODEL_FILENAMES} in that directory`);\n      }\n    }\n  }\n\n  const raw = readFileSync(join(dir, 'model.safetensors'));\n  const headerLen = Number(raw.readBigUInt64LE(0));\n  const header = JSON.parse(raw.subarray(8, 8 + headerLen).toString('utf8')) as Record<string, { dtype: string; shape: number[]; data_offsets: number[] }>;\n  const entry = Object.entries(header).find(([k]) => k !== '__metadata__');\n  if (!entry || entry[1].dtype !== 'F32') throw new SenseError('EMBED_MODEL', `${model}: expected an F32 safetensors matrix`);\n  const spec = entry[1];\n  const dims = spec.shape[1];\n  const dataStart = raw.byteOffset + 8 + headerLen + spec.data_offsets[0];\n  const matrix = dataStart % 4 === 0 ? new Float32Array(raw.buffer, dataStart, spec.shape[0] * dims) : new Float32Array(raw.buffer.slice(dataStart, dataStart + spec.shape[0] * dims * 4));\n\n  const tokenizerJson = JSON.parse(readFileSync(join(dir, 'tokenizer.json'), 'utf8'));\n  const { Tokenizer } = _require('@huggingface/tokenizers') as typeof import('@huggingface/tokenizers');\n  const tok = new Tokenizer(tokenizerJson, {});\n  const unkId = tokenizerJson.model?.vocab?.[tokenizerJson.model?.unk_token] ?? -1;\n\n  function one(text: string): Float32Array {\n    // The tokenizer yields undefined (not the unk id) for out-of-vocab tokens; unfiltered, it\n    // would index the matrix at NaN, and int8 conversion stores that NaN vector as all zeros.\n    const ids = (tok.encode(text, { add_special_tokens: false }).ids as number[]).filter((id) => Number.isInteger(id) && id !== unkId);\n    const v = new Float32Array(dims);\n    if (ids.length === 0) return v;\n    for (const id of ids) {\n      const off = id * dims;\n      for (let d = 0; d < dims; d++) v[d] += matrix[off + d];\n    }\n    let norm = 0;\n    for (let d = 0; d < dims; d++) {\n      v[d] /= ids.length;\n      norm += v[d] * v[d];\n    }\n    norm = Math.sqrt(norm) + 1e-32;\n    for (let d = 0; d < dims; d++) v[d] /= norm;\n    return v;\n  }\n\n  // Symmetric model: document and query embedding are the same call.\n  return {\n    id: `static:${model}`,\n    dims,\n    batchCap: BATCH_CAP,\n    languages,\n    embedDocuments: async (texts: string[], options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return texts.map(one);\n    },\n    embedQuery: async (text: string, options?: EmbedCallOptions) => {\n      options?.signal?.throwIfAborted();\n      return one(text);\n    },\n  };\n}\n"],"names":["staticProvider","BATCH_CAP","_require","require","Module","createRequire","model","root","tokenizerJson","dir","languages","cached","file","raw","headerLen","header","entry","spec","dims","dataStart","matrix","Tokenizer","tok","unkId","one","text","ids","encode","add_special_tokens","filter","id","Number","isInteger","v","Float32Array","length","off","d","norm","Math","sqrt","isDownloadable","downloadModel","into","console","error","readLanguages","undefined","modelDir","MODEL_FILES","existsSync","join","SenseError","MODEL_FILENAMES","readFileSync","readBigUInt64LE","JSON","parse","subarray","toString","Object","entries","find","k","dtype","shape","byteOffset","data_offsets","buffer","slice","vocab","unk_token","batchCap","embedDocuments","texts","options","signal","throwIfAborted","map","embedQuery"],"mappings":";;;;+BAesBA;;;eAAAA;;;sBAfmB;iEACtB;wBACE;wBACM;uBAC0E;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGrG,IAAMC,YAAY;AAElB,4FAA4F;AAC5F,2FAA2F;AAC3F,IAAMC,WAAW,OAAOC,YAAY,cAAcC,mBAAM,CAACC,aAAa,CAAC,uDAAmBF;AAInF,SAAeH,eAAeM,KAAa,EAAEC,IAAa;;kBA+BpBC,sBAA7BA,4BAAAA,uBA9BVC,KACAC,WAKIC,QAID,2BAAA,mBAAA,gBAAA,WAAA,OAAMC,MAOPC,KACAC,WACAC,QACAC,OAEAC,MACAC,MACAC,WACAC,QAEAZ,eACEa,WACFC,KACAC;QAEN,SAASC,IAAIC,IAAY;YACvB,0FAA0F;YAC1F,0FAA0F;YAC1F,IAAMC,MAAM,AAACJ,IAAIK,MAAM,CAACF,MAAM;gBAAEG,oBAAoB;YAAM,GAAGF,GAAG,CAAcG,MAAM,CAAC,SAACC;uBAAOC,OAAOC,SAAS,CAACF,OAAOA,OAAOP;;YAC5H,IAAMU,IAAI,IAAIC,aAAahB;YAC3B,IAAIQ,IAAIS,MAAM,KAAK,GAAG,OAAOF;gBACxB,kCAAA,2BAAA;;gBAAL,QAAK,YAAYP,wBAAZ,SAAA,6BAAA,QAAA,yBAAA,iCAAiB;oBAAjB,IAAMI,KAAN;oBACH,IAAMM,MAAMN,KAAKZ;oBACjB,IAAK,IAAImB,IAAI,GAAGA,IAAInB,MAAMmB,IAAKJ,CAAC,CAACI,EAAE,IAAIjB,MAAM,CAACgB,MAAMC,EAAE;gBACxD;;gBAHK;gBAAA;;;yBAAA,6BAAA;wBAAA;;;wBAAA;8BAAA;;;;YAIL,IAAIC,OAAO;YACX,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAK;gBAC7BJ,CAAC,CAACI,GAAE,IAAIX,IAAIS,MAAM;gBAClBG,QAAQL,CAAC,CAACI,GAAE,GAAGJ,CAAC,CAACI,GAAE;YACrB;YACAC,OAAOC,KAAKC,IAAI,CAACF,QAAQ;YACzB,IAAK,IAAID,KAAI,GAAGA,KAAInB,MAAMmB,KAAKJ,CAAC,CAACI,GAAE,IAAIC;YACvC,OAAOL;QACT;;;;yBAhDIQ,IAAAA,uBAAc,EAACnC,QAAfmC;;;;oBAEI;;wBAAMC,IAAAA,sBAAa,EAACpC,OAAO,SAACM,MAAM+B;mCAASC,QAAQC,KAAK,CAAC,AAAC,mBAA2BjC,OAATN,OAAM,KAAgBqC,OAAb/B,MAAK,UAAa,OAAL+B;2BAASpC;;;oBADjH,qFAAqF;oBACrFE,MAAM;oBACN,uFAAuF;oBACjFE,SAASmC,IAAAA,sBAAa,EAACxC,OAAOC;oBACpCG,YAAYC,UAAUA,OAAOwB,MAAM,GAAG,IAAIxB,SAASoC;;;;;;oBAEnDtC,MAAMuC,IAAAA,iBAAQ,EAAC1C,OAAOC;oBACjB,kCAAA,2BAAA;;wBAAL,IAAK,YAAc0C,oBAAW,uBAAzB,6BAAA,QAAA,yBAAA,iCAA2B;4BAArBrC,OAAN;4BACH,IAAI,CAACsC,IAAAA,kBAAU,EAACC,IAAAA,cAAI,EAAC1C,KAAKG,QAAQ;gCAChC,MAAM,IAAIwC,oBAAU,CAAC,uBAAuB,AAAC,eAAmD3C,OAArCH,OAAM,iCAAiD+C,OAAlB5C,KAAI,gBAA8B,OAAhB4C,wBAAe,EAAC;4BACpI;wBACF;;wBAJK;wBAAA;;;iCAAA,6BAAA;gCAAA;;;gCAAA;sCAAA;;;;;;oBAODxC,MAAMyC,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK;oBAC7BK,YAAYiB,OAAOlB,IAAI0C,eAAe,CAAC;oBACvCxC,SAASyC,KAAKC,KAAK,CAAC5C,IAAI6C,QAAQ,CAAC,GAAG,IAAI5C,WAAW6C,QAAQ,CAAC;oBAC5D3C,QAAQ4C,OAAOC,OAAO,CAAC9C,QAAQ+C,IAAI,CAAC;iEAAEC;+BAAOA,MAAM;;oBACzD,IAAI,CAAC/C,SAASA,KAAK,CAAC,EAAE,CAACgD,KAAK,KAAK,OAAO,MAAM,IAAIZ,oBAAU,CAAC,eAAe,AAAC,GAAQ,OAAN9C,OAAM;oBAC/EW,OAAOD,KAAK,CAAC,EAAE;oBACfE,OAAOD,KAAKgD,KAAK,CAAC,EAAE;oBACpB9C,YAAYN,IAAIqD,UAAU,GAAG,IAAIpD,YAAYG,KAAKkD,YAAY,CAAC,EAAE;oBACjE/C,SAASD,YAAY,MAAM,IAAI,IAAIe,aAAarB,IAAIuD,MAAM,EAAEjD,WAAWF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,QAAQ,IAAIgB,aAAarB,IAAIuD,MAAM,CAACC,KAAK,CAAClD,WAAWA,YAAYF,KAAKgD,KAAK,CAAC,EAAE,GAAG/C,OAAO;oBAE/KV,gBAAgBgD,KAAKC,KAAK,CAACH,IAAAA,oBAAY,EAACH,IAAAA,cAAI,EAAC1C,KAAK,mBAAmB;oBACnEY,YAAcnB,SAAS,2BAAvBmB;oBACFC,MAAM,IAAID,UAAUb,eAAe,CAAC;oBACpCe,iBAAQf,wBAAAA,cAAcF,KAAK,cAAnBE,6CAAAA,6BAAAA,sBAAqB8D,KAAK,cAA1B9D,iDAAAA,0BAA4B,EAACA,uBAAAA,cAAcF,KAAK,cAAnBE,2CAAAA,qBAAqB+D,SAAS,CAAC,uCAAI,CAAC;oBAsB/E,mEAAmE;oBACnE;;wBAAO;4BACLzC,IAAI,AAAC,UAAe,OAANxB;4BACdY,MAAAA;4BACAsD,UAAUvE;4BACVS,WAAAA;4BACA+D,gBAAgB,SAAhBA,eAAuBC,OAAiBC;;wCACtCA;;wCAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;wCAC/B;;4CAAOH,MAAMI,GAAG,CAACtD;;;gCACnB;;4BACAuD,YAAY,SAAZA,WAAmBtD,MAAckD;;wCAC/BA;;wCAAAA,oBAAAA,+BAAAA,kBAAAA,QAASC,MAAM,cAAfD,sCAAAA,gBAAiBE,cAAc;wCAC/B;;4CAAOrD,IAAIC;;;gCACb;;wBACF;;;;IACF"}

package/dist/cjs/embed/store.js

@@ -185,6 +185,10 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function _type_of(obj) {
+    "@swc/helpers - typeof";
+    return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj;
+}
 function modelPresent(cfg) {
     var e = (0, _indexts.embedConfig)(cfg);
     if (!e) return false;
@@ -197,7 +201,7 @@ function localModelMissing(e) {
 // removed, so only the first need for an id ever costs this HEAD request.
 function resolveSha(_0) {
     return _async_to_generator(function(model) {
-        var root, existing, url, res, err, sha;
+        var root, existing, url, res, sha;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -209,33 +213,19 @@ function resolveSha(_0) {
                         existing
                     ];
                     url = "https://huggingface.co/".concat(model, "/resolve/main/model.safetensors");
-                    _state.label = 1;
-                case 1:
-                    _state.trys.push([
-                        1,
-                        3,
-                        ,
-                        4
-                    ]);
                     return [
                         4,
                         (0, _httpts.fetchWithRetry)(url, {
                             method: 'HEAD',
                             redirect: 'manual'
+                        }, {
+                            attemptTimeoutMs: _httpts.HTTP_ATTEMPT_TIMEOUT_MS
                         })
                     ];
-                case 2:
+                case 1:
                     res = _state.sent();
-                    return [
-                        3,
-                        4
-                    ];
-                case 3:
-                    err = _state.sent();
-                    throw new _errorsts.SenseError('EMBED_MODEL', "could not reach ".concat(url, ": ").concat(err.message));
-                case 4:
                     sha = res.headers.get('x-repo-commit');
-                    if (!sha) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(url, ": response carried no x-repo-commit header"));
+                    if (!sha) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(res.url, ": response carried no x-repo-commit header"));
                     (0, _identityts.writeRef)(model, sha, root);
                     return [
                         2,
@@ -245,6 +235,19 @@ function resolveSha(_0) {
         });
     }).apply(this, arguments);
 }
+function isStringArray(value) {
+    return Array.isArray(value) && value.every(function(part) {
+        return typeof part === 'string';
+    });
+}
+function isHfModelInfo(value) {
+    if (value === null || (typeof value === "undefined" ? "undefined" : _type_of(value)) !== 'object') return false;
+    if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;
+    if (!('cardData' in value) || value.cardData === undefined) return true;
+    if (value.cardData === null || _type_of(value.cardData) !== 'object') return false;
+    if (!('language' in value.cardData) || value.cardData.language === undefined) return true;
+    return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);
+}
 // cardData.language is the structured field; tags are a fallback for cards that only tag
 // languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.
 function languagesFromCard(info) {
@@ -265,7 +268,7 @@ function cardUnreadable(model) {
 }
 function ensureLanguages(_0) {
     return _async_to_generator(function(model) {
-        var root, _languagesFromCard, measured, res, unused, info, unused1;
+        var root, _languagesFromCard, measured, res, unused, info, decoded;
         var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
@@ -292,7 +295,9 @@ function ensureLanguages(_0) {
                     ]);
                     return [
                         4,
-                        (0, _httpts.fetchWithRetry)("https://huggingface.co/api/models/".concat(model), {})
+                        (0, _httpts.fetchWithRetry)("https://huggingface.co/api/models/".concat(model), {}, {
+                            attemptTimeoutMs: _httpts.HTTP_ATTEMPT_TIMEOUT_MS
+                        })
                     ];
                 case 2:
                     res = _state.sent();
@@ -313,31 +318,16 @@ function ensureLanguages(_0) {
                             2
                         ];
                     }
-                    _state.label = 5;
-                case 5:
-                    _state.trys.push([
-                        5,
-                        7,
-                        ,
-                        8
-                    ]);
-                    return [
-                        4,
-                        res.json()
-                    ];
-                case 6:
-                    info = _state.sent();
-                    return [
-                        3,
-                        8
-                    ];
-                case 7:
-                    unused1 = _state.sent();
-                    cardUnreadable(model);
-                    return [
-                        2
-                    ];
-                case 8:
+                    try {
+                        decoded = JSON.parse(new TextDecoder().decode(res.body));
+                        if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');
+                        info = decoded;
+                    } catch (unused) {
+                        cardUnreadable(model);
+                        return [
+                            2
+                        ];
+                    }
                     (0, _identityts.writeLanguages)(model, (_languagesFromCard = languagesFromCard(info)) !== null && _languagesFromCard !== void 0 ? _languagesFromCard : [], root);
                     return [
                         2
@@ -348,45 +338,20 @@ function ensureLanguages(_0) {
 }
 function fetchToFile(url, dest) {
     return _async_to_generator(function() {
-        var res, err, _tmp, _;
+        var res;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    _state.trys.push([
-                        0,
-                        2,
-                        ,
-                        3
-                    ]);
                     return [
                         4,
-                        (0, _httpts.fetchWithRetry)(url, {})
+                        (0, _httpts.fetchWithRetry)(url, {}, {
+                            attemptTimeoutMs: _httpts.MODEL_FILE_ATTEMPT_TIMEOUT_MS
+                        })
                     ];
                 case 1:
                     res = _state.sent();
-                    return [
-                        3,
-                        3
-                    ];
-                case 2:
-                    err = _state.sent();
-                    throw new _errorsts.SenseError('EMBED_MODEL', "could not reach ".concat(url, ": ").concat(err.message));
-                case 3:
-                    if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(url, " -> HTTP ").concat(res.status));
-                    _tmp = [
-                        dest
-                    ];
-                    _ = Buffer.from;
-                    return [
-                        4,
-                        res.arrayBuffer()
-                    ];
-                case 4:
-                    _atomicwritets.writeFileAtomic.apply(void 0, _tmp.concat([
-                        _.apply(Buffer, [
-                            _state.sent()
-                        ])
-                    ]));
+                    if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(res.url, " -> HTTP ").concat(res.status));
+                    (0, _atomicwritets.writeFileAtomic)(dest, Buffer.from(res.body));
                     return [
                         2
                     ];

package/dist/cjs/embed/store.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { fetchWithRetry } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' });\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: Response;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {});\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    info = (await res.json()) as HfModelInfo;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  let res: Response;\n  try {\n    res = await fetchWithRetry(url, {});\n  } catch (err) {\n    throw new SenseError('EMBED_MODEL', `could not reach ${url}: ${(err as Error).message}`);\n  }\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(await res.arrayBuffer()));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","downloadModel","downloadModelRevision","hasModelFiles","isDownloadable","localModelMissing","modelDir","modelPresent","readLanguages","refsMainPath","snapshotDir","writeLanguages","writeRef","cfg","e","embedConfig","provider","model","resolveSha","root","existing","url","res","err","sha","DEFAULT_MODELS_ROOT","readRef","fetchWithRetry","method","redirect","SenseError","message","headers","get","languagesFromCard","info","lang","cardData","language","Array","isArray","length","tagLangs","tags","filter","isKnownLanguageTag","undefined","cardUnreadable","console","error","ensureLanguages","measured","existsSync","languagesPath","MEASURED_MODEL_LANGUAGES","ok","json","fetchToFile","dest","status","Buffer","from","arrayBuffer","writeFileAtomic","onFile","dir","file","mkdirSync","recursive","join"],"mappings":";;;;;;;;;;;QAYwCA;eAAAA,2BAAe;;QAAEC;eAAAA,uBAAW;;QAkH9CC;eAAAA;;QAbAC;eAAAA;;QArGbC;eAAAA,yBAAa;;QAAEC;eAAAA,0BAAc;;QActBC;eAAAA;;QAdsDC;eAAAA,oBAAQ;;QAI9DC;eAAAA;;QAJgEC;eAAAA,yBAAa;;QAAEC;eAAAA,wBAAY;;QAAEC;eAAAA,uBAAW;;QAAEC;eAAAA,0BAAc;;QAAEC;eAAAA,oBAAQ;;;sBAZ5G;wBACjB;uBAEO;wBACD;6BACK;sBACD;0BACgH;2BAClF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQtD,SAASL,aAAaM,GAAW;IACtC,IAAMC,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,QAAQ,KAAK,YAAYb,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACzD;AAMO,SAASZ,kBAAkBS,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEE,QAAQ,KAAK,YAAY,CAACZ,IAAAA,0BAAc,EAACU,EAAEG,KAAK,KAAK,CAACd,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,SAAeC;wCAAWD,KAAa;YAAEE,MACjCC,UAEAC,KAGFC,KAGKC,KAGHC;;;;;oBAZiCL,OAAAA,oEAAeM,+BAAmB;oBACnEL,WAAWM,IAAAA,mBAAO,EAACT,OAAOE;oBAChC,IAAIC,UAAU;;wBAAOA;;oBACfC,MAAM,AAAC,0BAA+B,OAANJ,OAAM;;;;;;;;;oBAKpC;;wBAAMU,IAAAA,sBAAc,EAACN,KAAK;4BAAEO,QAAQ;4BAAQC,UAAU;wBAAS;;;oBAArEP,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIO,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAART,KAAI,MAA2B,OAAvB,AAACE,IAAcQ,OAAO;;oBAEjFP,MAAMF,IAAIU,OAAO,CAACC,GAAG,CAAC;oBAC5B,IAAI,CAACT,KAAK,MAAM,IAAIM,oBAAU,CAAC,eAAe,AAAC,GAAM,OAAJT,KAAI;oBACrDT,IAAAA,oBAAQ,EAACK,OAAOO,KAAKL;oBACrB;;wBAAOK;;;;IACT;;AAOA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASU,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,IAAMC,QAAOD,iBAAAA,KAAKE,QAAQ,cAAbF,qCAAAA,eAAeG,QAAQ;IACpC,IAAI,OAAOF,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIG,MAAMC,OAAO,CAACJ,SAASA,KAAKK,MAAM,GAAG,GAAG,OAAOL;IACnD,IAAMM,WAAW,EAACP,aAAAA,KAAKQ,IAAI,cAATR,wBAAAA,aAAa,EAAE,EAAES,MAAM,CAACC,+BAAkB;IAC5D,OAAOH,SAASD,MAAM,GAAG,IAAIC,WAAWI;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASC,eAAe9B,KAAa;IACnC+B,QAAQC,KAAK,CAAC,AAAC,4CAAiD,OAANhC,OAAM;AAClE;AAEA,SAAeiC;wCAAgBjC,KAAa;YAAEE,MA0BtBe,oBAvBhBiB,UAKF7B,aAWAa;;;;;oBAnBwChB,OAAAA,oEAAeM,+BAAmB;oBAC9E,IAAI2B,IAAAA,kBAAU,EAACC,IAAAA,yBAAa,EAACpC,OAAOE,QAAQ;;;oBAC5C,kFAAkF;oBAC5EgC,WAAWG,qCAAwB,CAACrC,MAAM;oBAChD,IAAIkC,UAAU;wBACZxC,IAAAA,0BAAc,EAACM,OAAOkC,UAAUhC;wBAChC;;;oBACF;;;;;;;;;oBAGQ;;wBAAMQ,IAAAA,sBAAc,EAAC,AAAC,qCAA0C,OAANV,QAAS,CAAC;;;oBAA1EK,MAAM;;;;;;;oBAENyB,eAAe9B;oBACf;;;;oBAEF,IAAI,CAACK,IAAIiC,EAAE,EAAE;wBACXR,eAAe9B;wBACf;;;oBACF;;;;;;;;;oBAGU;;wBAAMK,IAAIkC,IAAI;;;oBAAtBrB,OAAQ;;;;;;;oBAERY,eAAe9B;oBACf;;;;oBAEFN,IAAAA,0BAAc,EAACM,QAAOiB,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,yBAA+Bf;;;;;;IACvD;;AAEA,SAAesC,YAAYpC,GAAW,EAAEqC,IAAY;;YAC9CpC,KAGKC;;;;;;;;;;oBADD;;wBAAMI,IAAAA,sBAAc,EAACN,KAAK,CAAC;;;oBAAjCC,MAAM;;;;;;oBACCC;oBACP,MAAM,IAAIO,oBAAU,CAAC,eAAe,AAAC,mBAA0B,OAART,KAAI,MAA2B,OAAvB,AAACE,IAAcQ,OAAO;;oBAEvF,IAAI,CAACT,IAAIiC,EAAE,EAAE,MAAM,IAAIzB,oBAAU,CAAC,eAAe,AAAC,GAAiBR,OAAfD,KAAI,aAAsB,OAAXC,IAAIqC,MAAM;;wBAC7DD;;wBAAME,OAAOC,IAAI;oBAAC;;wBAAMvC,IAAIwC,WAAW;;;oBAAvDC,8BAAe;wBAAOH,QAAAA;4BAAY;;;;;;;;IACpC;;AAIO,SAAe1D;wCAAsBe,KAAa,EAAEO,GAAW,EAAEwC,MAA4C;YAAE7C,MAC9G8C,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;;oBAHyG/C,OAAAA,oEAAeM,+BAAmB;oBAChJwC,MAAMvD,IAAAA,uBAAW,EAACO,OAAOO,KAAKL;oBACpCgD,IAAAA,iBAAS,EAACF,KAAK;wBAAEG,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAcpE,uBAAW;;;2BAAzB,6BAAA,QAAA;;;;oBAAMkE,OAAN;oBACH,IAAId,IAAAA,kBAAU,EAACiB,IAAAA,cAAI,EAACJ,KAAKC,QAAQ;;;;oBACjCF,mBAAAA,6BAAAA,OAASE,MAAMD;oBACf;;wBAAMR,YAAY,AAAC,0BAA0CjC,OAAjBP,OAAM,aAAkBiD,OAAP1C,KAAI,KAAQ,OAAL0C,OAAQG,IAAAA,cAAI,EAACJ,KAAKC;;;oBAAtF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOD;;;;IACT;;AAIO,SAAehE;wCAAcgB,KAAa,EAAE+C,MAA4C;YAAE7C,MAIzFK;;;;;oBAJyFL,OAAAA,oEAAeM,+BAAmB;oBACjI,IAAI,CAACrB,IAAAA,0BAAc,EAACa,QAAQ;wBAC1B,MAAM,IAAIa,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANb,OAAM;oBAC5D;oBACY;;wBAAMC,WAAWD,OAAOE;;;oBAA9BK,MAAM;oBACZ,sFAAsF;oBACtF,oFAAoF;oBACpF;;wBAAM0B,gBAAgBjC,OAAOE;;;oBAA7B;oBACA;;wBAAOjB,sBAAsBe,OAAOO,KAAKwC,QAAQ7C;;;;IACnD"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/store.ts"],"sourcesContent":["import { existsSync, mkdirSync } from 'node:fs';\nimport { join } from 'node:path';\nimport type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { writeFileAtomic } from '../lib/atomic-write.ts';\nimport { type BufferedResponse, fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS, MODEL_FILE_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport { DEFAULT_MODELS_ROOT, hasModelFiles, isDownloadable, languagesPath, MODEL_FILES, readRef, snapshotDir, writeLanguages, writeRef } from './identity.ts';\nimport { isKnownLanguageTag, MEASURED_MODEL_LANGUAGES } from './languages.ts';\n\n// identity.ts holds paths and cache identity, no network; store.ts holds the network fetches\n// (downloads, sha resolution, language metadata), re-exported here for its network surface.\nexport { hasModelFiles, isDownloadable, MODEL_FILENAMES, MODEL_FILES, modelDir, readLanguages, refsMainPath, snapshotDir, writeLanguages, writeRef } from './identity.ts';\n\n// A non-static provider has nothing to download here; an unreachable endpoint surfaces as an\n// EMBED_MODEL error at call time instead.\nexport function modelPresent(cfg: Config): boolean {\n  const e = embedConfig(cfg);\n  if (!e) return false;\n  return e.provider !== 'static' || hasModelFiles(e.model);\n}\n\ntype ResolvedEmbedConfig = NonNullable<ReturnType<typeof embedConfig>>;\n\n// True when embed.model names a local path (not a Hugging Face id) that is missing its model\n// files -- nothing will ever fetch it for itself, unlike a downloadable id.\nexport function localModelMissing(e: ResolvedEmbedConfig | null): e is ResolvedEmbedConfig {\n  return e !== null && e.provider === 'static' && !isDownloadable(e.model) && !hasModelFiles(e.model);\n}\n\n// `main` resolves to a sha once per model: a recorded ref pins until its snapshot is\n// removed, so only the first need for an id ever costs this HEAD request.\nasync function resolveSha(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const existing = readRef(model, root);\n  if (existing) return existing;\n  const url = `https://huggingface.co/${model}/resolve/main/model.safetensors`;\n  // redirect: manual -- the header rides the Hub's own response; following the LFS/CDN\n  // redirect would read the CDN's headers instead, which do not carry it.\n  const res = await fetchWithRetry(url, { method: 'HEAD', redirect: 'manual' }, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  const sha = res.headers.get('x-repo-commit');\n  if (!sha) throw new SenseError('EMBED_MODEL', `${res.url}: response carried no x-repo-commit header`);\n  writeRef(model, sha, root);\n  return sha;\n}\n\ninterface HfModelInfo {\n  cardData?: { language?: string | string[] };\n  tags?: string[];\n}\n\nfunction isStringArray(value: unknown): value is string[] {\n  return Array.isArray(value) && value.every((part) => typeof part === 'string');\n}\n\nfunction isHfModelInfo(value: unknown): value is HfModelInfo {\n  if (value === null || typeof value !== 'object') return false;\n  if ('tags' in value && value.tags !== undefined && !isStringArray(value.tags)) return false;\n  if (!('cardData' in value) || value.cardData === undefined) return true;\n  if (value.cardData === null || typeof value.cardData !== 'object') return false;\n  if (!('language' in value.cardData) || value.cardData.language === undefined) return true;\n  return typeof value.cardData.language === 'string' || isStringArray(value.cardData.language);\n}\n\n// cardData.language is the structured field; tags are a fallback for cards that only tag\n// languages, filtered to recognized codes so an unrelated short tag is never mistaken for one.\nfunction languagesFromCard(info: HfModelInfo): string[] | undefined {\n  const lang = info.cardData?.language;\n  if (typeof lang === 'string') return [lang];\n  if (Array.isArray(lang) && lang.length > 0) return lang;\n  const tagLangs = (info.tags ?? []).filter(isKnownLanguageTag);\n  return tagLangs.length > 0 ? tagLangs : undefined;\n}\n\n// Unreachable metadata is not an error, only the language-fit check staying off, but it says so:\n// the check that keeps a misfit model loud cannot itself go quiet (PRINCIPLES: no-silent-modes).\nfunction cardUnreadable(model: string): void {\n  console.error(`sense: could not read the model card for ${model}; the language-fit check is off until it resolves (retries on next run, or run \\`sense download\\`)`);\n}\n\nasync function ensureLanguages(model: string, root: string = DEFAULT_MODELS_ROOT): Promise<void> {\n  if (existsSync(languagesPath(model, root))) return;\n  // A measured entry needs no card and no network, and stays deterministic offline.\n  const measured = MEASURED_MODEL_LANGUAGES[model];\n  if (measured) {\n    writeLanguages(model, measured, root);\n    return;\n  }\n  let res: BufferedResponse;\n  try {\n    res = await fetchWithRetry(`https://huggingface.co/api/models/${model}`, {}, { attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  if (!res.ok) {\n    cardUnreadable(model);\n    return;\n  }\n  let info: HfModelInfo;\n  try {\n    const decoded: unknown = JSON.parse(new TextDecoder().decode(res.body));\n    if (!isHfModelInfo(decoded)) throw new Error('invalid model-card response');\n    info = decoded;\n  } catch {\n    cardUnreadable(model);\n    return;\n  }\n  writeLanguages(model, languagesFromCard(info) ?? [], root);\n}\n\nasync function fetchToFile(url: string, dest: string): Promise<void> {\n  const res = await fetchWithRetry(url, {}, { attemptTimeoutMs: MODEL_FILE_ATTEMPT_TIMEOUT_MS });\n  if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n  writeFileAtomic(dest, Buffer.from(res.body));\n}\n\n// Fetches an already-known sha straight into its snapshot dir, without touching refs/main --\n// for downloadModel below, and for callers pinning an exact revision (the parity suite).\nexport async function downloadModelRevision(model: string, sha: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  const dir = snapshotDir(model, sha, root);\n  mkdirSync(dir, { recursive: true });\n  for (const file of MODEL_FILES) {\n    if (existsSync(join(dir, file))) continue;\n    onFile?.(file, dir);\n    await fetchToFile(`https://huggingface.co/${model}/resolve/${sha}/${file}`, join(dir, file));\n  }\n  return dir;\n}\n\n// The only code path that resolves `main`: a static provider's first construction calls it\n// lazily, `sense download` as an explicit prefetch. Idempotent either way.\nexport async function downloadModel(model: string, onFile?: (file: string, dir: string) => void, root: string = DEFAULT_MODELS_ROOT): Promise<string> {\n  if (!isDownloadable(model)) {\n    throw new SenseError('EMBED_MODEL', `embed.model \"${model}\" is a local path, not a Hugging Face model id, so there is nothing to download; put model.safetensors and tokenizer.json in that directory, or name a model id like \"minishlab/potion-retrieval-32M\"`);\n  }\n  const sha = await resolveSha(model, root);\n  // Independent of the weights above: a fresh HEAD/GET pair either way, so a lazy first\n  // construction and an explicit prefetch both end up with a cached language verdict.\n  await ensureLanguages(model, root);\n  return downloadModelRevision(model, sha, onFile, root);\n}\n"],"names":["MODEL_FILENAMES","MODEL_FILES","downloadModel","downloadModelRevision","hasModelFiles","isDownloadable","localModelMissing","modelDir","modelPresent","readLanguages","refsMainPath","snapshotDir","writeLanguages","writeRef","cfg","e","embedConfig","provider","model","resolveSha","root","existing","url","res","sha","DEFAULT_MODELS_ROOT","readRef","fetchWithRetry","method","redirect","attemptTimeoutMs","HTTP_ATTEMPT_TIMEOUT_MS","headers","get","SenseError","isStringArray","value","Array","isArray","every","part","isHfModelInfo","tags","undefined","cardData","language","languagesFromCard","info","lang","length","tagLangs","filter","isKnownLanguageTag","cardUnreadable","console","error","ensureLanguages","measured","decoded","existsSync","languagesPath","MEASURED_MODEL_LANGUAGES","ok","JSON","parse","TextDecoder","decode","body","Error","fetchToFile","dest","MODEL_FILE_ATTEMPT_TIMEOUT_MS","status","writeFileAtomic","Buffer","from","onFile","dir","file","mkdirSync","recursive","join"],"mappings":";;;;;;;;;;;QAYwCA;eAAAA,2BAAe;;QAAEC;eAAAA,uBAAW;;QAuH9CC;eAAAA;;QAbAC;eAAAA;;QA1GbC;eAAAA,yBAAa;;QAAEC;eAAAA,0BAAc;;QActBC;eAAAA;;QAdsDC;eAAAA,oBAAQ;;QAI9DC;eAAAA;;QAJgEC;eAAAA,yBAAa;;QAAEC;eAAAA,wBAAY;;QAAEC;eAAAA,uBAAW;;QAAEC;eAAAA,0BAAc;;QAAEC;eAAAA,oBAAQ;;;sBAZ5G;wBACjB;uBAEO;wBACD;6BACK;sBAC8E;0BACiC;2BAClF;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAQtD,SAASL,aAAaM,GAAW;IACtC,IAAMC,IAAIC,IAAAA,oBAAW,EAACF;IACtB,IAAI,CAACC,GAAG,OAAO;IACf,OAAOA,EAAEE,QAAQ,KAAK,YAAYb,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACzD;AAMO,SAASZ,kBAAkBS,CAA6B;IAC7D,OAAOA,MAAM,QAAQA,EAAEE,QAAQ,KAAK,YAAY,CAACZ,IAAAA,0BAAc,EAACU,EAAEG,KAAK,KAAK,CAACd,IAAAA,yBAAa,EAACW,EAAEG,KAAK;AACpG;AAEA,qFAAqF;AACrF,0EAA0E;AAC1E,SAAeC;wCAAWD,KAAa;YAAEE,MACjCC,UAEAC,KAGAC,KACAC;;;;;oBAPiCJ,OAAAA,oEAAeK,+BAAmB;oBACnEJ,WAAWK,IAAAA,mBAAO,EAACR,OAAOE;oBAChC,IAAIC,UAAU;;wBAAOA;;oBACfC,MAAM,AAAC,0BAA+B,OAANJ,OAAM;oBAGhC;;wBAAMS,IAAAA,sBAAc,EAACL,KAAK;4BAAEM,QAAQ;4BAAQC,UAAU;wBAAS,GAAG;4BAAEC,kBAAkBC,+BAAuB;wBAAC;;;oBAApHR,MAAM;oBACNC,MAAMD,IAAIS,OAAO,CAACC,GAAG,CAAC;oBAC5B,IAAI,CAACT,KAAK,MAAM,IAAIU,oBAAU,CAAC,eAAe,AAAC,GAAU,OAARX,IAAID,GAAG,EAAC;oBACzDT,IAAAA,oBAAQ,EAACK,OAAOM,KAAKJ;oBACrB;;wBAAOI;;;;IACT;;AAOA,SAASW,cAAcC,KAAc;IACnC,OAAOC,MAAMC,OAAO,CAACF,UAAUA,MAAMG,KAAK,CAAC,SAACC;eAAS,OAAOA,SAAS;;AACvE;AAEA,SAASC,cAAcL,KAAc;IACnC,IAAIA,UAAU,QAAQ,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,UAAU,OAAO;IACxD,IAAI,UAAUA,SAASA,MAAMM,IAAI,KAAKC,aAAa,CAACR,cAAcC,MAAMM,IAAI,GAAG,OAAO;IACtF,IAAI,CAAE,CAAA,cAAcN,KAAI,KAAMA,MAAMQ,QAAQ,KAAKD,WAAW,OAAO;IACnE,IAAIP,MAAMQ,QAAQ,KAAK,QAAQ,SAAOR,MAAMQ,QAAQ,MAAK,UAAU,OAAO;IAC1E,IAAI,CAAE,CAAA,cAAcR,MAAMQ,QAAQ,AAAD,KAAMR,MAAMQ,QAAQ,CAACC,QAAQ,KAAKF,WAAW,OAAO;IACrF,OAAO,OAAOP,MAAMQ,QAAQ,CAACC,QAAQ,KAAK,YAAYV,cAAcC,MAAMQ,QAAQ,CAACC,QAAQ;AAC7F;AAEA,yFAAyF;AACzF,+FAA+F;AAC/F,SAASC,kBAAkBC,IAAiB;QAIxBA;QAHLA;IAAb,IAAMC,QAAOD,iBAAAA,KAAKH,QAAQ,cAAbG,qCAAAA,eAAeF,QAAQ;IACpC,IAAI,OAAOG,SAAS,UAAU,OAAO;QAACA;KAAK;IAC3C,IAAIX,MAAMC,OAAO,CAACU,SAASA,KAAKC,MAAM,GAAG,GAAG,OAAOD;IACnD,IAAME,WAAW,EAACH,aAAAA,KAAKL,IAAI,cAATK,wBAAAA,aAAa,EAAE,EAAEI,MAAM,CAACC,+BAAkB;IAC5D,OAAOF,SAASD,MAAM,GAAG,IAAIC,WAAWP;AAC1C;AAEA,iGAAiG;AACjG,iGAAiG;AACjG,SAASU,eAAenC,KAAa;IACnCoC,QAAQC,KAAK,CAAC,AAAC,4CAAiD,OAANrC,OAAM;AAClE;AAEA,SAAesC;wCAAgBtC,KAAa;YAAEE,MA4BtB0B,oBAzBhBW,UAKFlC,aAWAwB,MAEIW;;;;;oBArBoCtC,OAAAA,oEAAeK,+BAAmB;oBAC9E,IAAIkC,IAAAA,kBAAU,EAACC,IAAAA,yBAAa,EAAC1C,OAAOE,QAAQ;;;oBAC5C,kFAAkF;oBAC5EqC,WAAWI,qCAAwB,CAAC3C,MAAM;oBAChD,IAAIuC,UAAU;wBACZ7C,IAAAA,0BAAc,EAACM,OAAOuC,UAAUrC;wBAChC;;;oBACF;;;;;;;;;oBAGQ;;wBAAMO,IAAAA,sBAAc,EAAC,AAAC,qCAA0C,OAANT,QAAS,CAAC,GAAG;4BAAEY,kBAAkBC,+BAAuB;wBAAC;;;oBAAzHR,MAAM;;;;;;;oBAEN8B,eAAenC;oBACf;;;;oBAEF,IAAI,CAACK,IAAIuC,EAAE,EAAE;wBACXT,eAAenC;wBACf;;;oBACF;oBAEA,IAAI;wBACIwC,UAAmBK,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAAC3C,IAAI4C,IAAI;wBACrE,IAAI,CAAC1B,cAAciB,UAAU,MAAM,IAAIU,MAAM;wBAC7CrB,OAAOW;oBACT,EAAE,eAAM;wBACNL,eAAenC;wBACf;;;oBACF;oBACAN,IAAAA,0BAAc,EAACM,QAAO4B,qBAAAA,kBAAkBC,mBAAlBD,gCAAAA,yBAA+B1B;;;;;;IACvD;;AAEA,SAAeiD,YAAY/C,GAAW,EAAEgD,IAAY;;YAC5C/C;;;;oBAAM;;wBAAMI,IAAAA,sBAAc,EAACL,KAAK,CAAC,GAAG;4BAAEQ,kBAAkByC,qCAA6B;wBAAC;;;oBAAtFhD,MAAM;oBACZ,IAAI,CAACA,IAAIuC,EAAE,EAAE,MAAM,IAAI5B,oBAAU,CAAC,eAAe,AAAC,GAAqBX,OAAnBA,IAAID,GAAG,EAAC,aAAsB,OAAXC,IAAIiD,MAAM;oBACjFC,IAAAA,8BAAe,EAACH,MAAMI,OAAOC,IAAI,CAACpD,IAAI4C,IAAI;;;;;;IAC5C;;AAIO,SAAehE;wCAAsBe,KAAa,EAAEM,GAAW,EAAEoD,MAA4C;YAAExD,MAC9GyD,KAED,2BAAA,mBAAA,gBAAA,WAAA,OAAMC;;;;;oBAHyG1D,OAAAA,oEAAeK,+BAAmB;oBAChJoD,MAAMlE,IAAAA,uBAAW,EAACO,OAAOM,KAAKJ;oBACpC2D,IAAAA,iBAAS,EAACF,KAAK;wBAAEG,WAAW;oBAAK;oBAC5B,kCAAA,2BAAA;;;;;;;;;oBAAA,YAAc/E,uBAAW;;;2BAAzB,6BAAA,QAAA;;;;oBAAM6E,OAAN;oBACH,IAAInB,IAAAA,kBAAU,EAACsB,IAAAA,cAAI,EAACJ,KAAKC,QAAQ;;;;oBACjCF,mBAAAA,6BAAAA,OAASE,MAAMD;oBACf;;wBAAMR,YAAY,AAAC,0BAA0C7C,OAAjBN,OAAM,aAAkB4D,OAAPtD,KAAI,KAAQ,OAALsD,OAAQG,IAAAA,cAAI,EAACJ,KAAKC;;;oBAAtF;;;oBAHG;;;;;;;;;;;;oBAAA;oBAAA;;;;;;;6BAAA,6BAAA;4BAAA;;;4BAAA;kCAAA;;;;;;;oBAKL;;wBAAOD;;;;IACT;;AAIO,SAAe3E;wCAAcgB,KAAa,EAAE0D,MAA4C;YAAExD,MAIzFI;;;;;oBAJyFJ,OAAAA,oEAAeK,+BAAmB;oBACjI,IAAI,CAACpB,IAAAA,0BAAc,EAACa,QAAQ;wBAC1B,MAAM,IAAIgB,oBAAU,CAAC,eAAe,AAAC,gBAAqB,OAANhB,OAAM;oBAC5D;oBACY;;wBAAMC,WAAWD,OAAOE;;;oBAA9BI,MAAM;oBACZ,sFAAsF;oBACtF,oFAAoF;oBACpF;;wBAAMgC,gBAAgBtC,OAAOE;;;oBAA7B;oBACA;;wBAAOjB,sBAAsBe,OAAOM,KAAKoD,QAAQxD;;;;IACnD"}

package/dist/cjs/embed/types.d.cts

@@ -1,9 +1,12 @@
 export declare const STORE_DIMS = 256;
+export interface EmbedCallOptions {
+    signal?: AbortSignal;
+}
 export interface EmbedProvider {
     id: string;
     dims: number;
     batchCap: number;
     languages?: string[];
-    embedDocuments(texts: string[]): Promise<Float32Array[]>;
-    embedQuery(text: string): Promise<Float32Array>;
+    embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;
+    embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;
 }

package/dist/cjs/embed/types.d.ts

@@ -1,9 +1,12 @@
 export declare const STORE_DIMS = 256;
+export interface EmbedCallOptions {
+    signal?: AbortSignal;
+}
 export interface EmbedProvider {
     id: string;
     dims: number;
     batchCap: number;
     languages?: string[];
-    embedDocuments(texts: string[]): Promise<Float32Array[]>;
-    embedQuery(text: string): Promise<Float32Array>;
+    embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;
+    embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;
 }

package/dist/cjs/embed/types.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[]): Promise<Float32Array[]>;\n  embedQuery(text: string): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;;;;;+BACrFA;;;eAAAA;;;AAAN,IAAMA,aAAa"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/types.ts"],"sourcesContent":["// Storage lever fixed by the bake-off (benchmark/reports/2026-08-13-static-model-bakeoff.md):\n// int8 at 256 dims is quality-free vs f32-512 when fused; also the duckdb store's FLOAT[N] width.\nexport const STORE_DIMS = 256;\n\nexport interface EmbedCallOptions {\n  signal?: AbortSignal;\n}\n\n// The provider contract every wire protocol implements: a Liskov-substitutable\n// surface so query/search code never branches on provider identity.\nexport interface EmbedProvider {\n  id: string; // model identity; participates in the cache key, change -> re-embed\n  dims: number;\n  batchCap: number; // max texts per embedDocuments call\n  languages?: string[]; // declared model languages, for the language-fit check\n  embedDocuments(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]>;\n  embedQuery(text: string, options?: EmbedCallOptions): Promise<Float32Array>;\n}\n"],"names":["STORE_DIMS"],"mappings":"AAAA,8FAA8F;AAC9F,kGAAkG;;;;;+BACrFA;;;eAAAA;;;AAAN,IAAMA,aAAa"}