sensemaking 0.25.1 → 0.27.0

package/dist/esm/embed/http.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    try {\n      const res = await fetch(url, init);\n      if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n      const retryAfter = Number(res.headers.get('retry-after'));\n      // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n      await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n    } catch (err) {\n      if (attempt === MAX_ATTEMPTS - 1) throw err;\n      await sleep(baseDelayMs * 2 ** attempt);\n    }\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n"],"names":["MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","fetchWithRetry","url","init","baseDelayMs","attempt","res","fetch","ok","status","retryAfter","Number","headers","get","Math","min","err","Error","probeReachable","timeoutMs","signal","AbortSignal","timeout"],"mappings":"AAAA,MAAMA,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,2FAA2F;AAC3F,+FAA+F;AAC/F,OAAO,eAAeI,eAAeC,GAAW,EAAEC,IAAiB,EAAEC,cAAc,GAAG;IACpF,IAAK,IAAIC,UAAU,GAAGA,UAAUV,cAAcU,UAAW;QACvD,IAAI;YACF,MAAMC,MAAM,MAAMC,MAAML,KAAKC;YAC7B,IAAIG,IAAIE,EAAE,IAAKF,IAAIG,MAAM,KAAK,OAAOH,IAAIG,MAAM,GAAG,OAAQJ,YAAYV,eAAe,GAAG,OAAOW;YAC/F,MAAMI,aAAaC,OAAOL,IAAIM,OAAO,CAACC,GAAG,CAAC;YAC1C,iFAAiF;YACjF,MAAMjB,MAAMc,aAAa,IAAII,KAAKC,GAAG,CAACL,aAAa,MAAM,SAAUN,cAAc,KAAKC;QACxF,EAAE,OAAOW,KAAK;YACZ,IAAIX,YAAYV,eAAe,GAAG,MAAMqB;YACxC,MAAMpB,MAAMQ,cAAc,KAAKC;QACjC;IACF;IACA,MAAM,IAAIY,MAAM;AAClB;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,OAAO,eAAeC,eAAehB,GAAW,EAAEiB,YAAY,IAAI;IAChE,IAAI;QACF,MAAMZ,MAAML,KAAK;YAAEkB,QAAQC,YAAYC,OAAO,CAACH;QAAW;QAC1D,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\nconst MAX_ATTEMPTS = 3;\nconst MAX_RETRY_AFTER_MS = 10_000;\nconst DEFAULT_BASE_DELAY_MS = 200;\n\n// Initial policy bounds small API/metadata responses separately from model-file downloads.\nexport const HTTP_ATTEMPT_TIMEOUT_MS = 30_000;\nexport const MODEL_FILE_ATTEMPT_TIMEOUT_MS = 5 * 60_000;\n\nexport type FetchRequestInit = Omit<RequestInit, 'signal'> & { signal?: never };\n\nexport interface FetchRetryOptions {\n  signal?: AbortSignal;\n  attemptTimeoutMs?: number;\n  baseDelayMs?: number;\n}\n\nexport interface BufferedResponse {\n  readonly ok: boolean;\n  readonly status: number;\n  readonly headers: Headers;\n  readonly body: ArrayBuffer;\n  readonly url: string;\n}\n\ntype ResponsePhase = 'response headers' | 'response body';\n\nfunction safeUrl(url: string): string {\n  try {\n    const parsed = new URL(url);\n    parsed.username = '';\n    parsed.password = '';\n    parsed.search = '';\n    parsed.hash = '';\n    return parsed.href;\n  } catch {\n    return 'the configured embedding endpoint';\n  }\n}\n\nfunction attemptController(\n  callerSignal: AbortSignal | undefined,\n  timeoutMs: number\n): {\n  controller: AbortController;\n  timeoutReason: object;\n  cleanup: () => void;\n} {\n  callerSignal?.throwIfAborted();\n  const controller = new AbortController();\n  const timeoutReason = {};\n  const relayAbort = () => controller.abort(callerSignal?.reason);\n  callerSignal?.addEventListener('abort', relayAbort, { once: true });\n  if (callerSignal?.aborted) relayAbort();\n  const timer = setTimeout(() => controller.abort(timeoutReason), timeoutMs);\n  return {\n    controller,\n    timeoutReason,\n    cleanup: () => {\n      clearTimeout(timer);\n      callerSignal?.removeEventListener('abort', relayAbort);\n    },\n  };\n}\n\nfunction disposeResponse(response: Response, controller: AbortController): void {\n  const cancellation = response.body && !response.bodyUsed ? response.body.cancel() : undefined;\n  controller.abort();\n  // The controller owns socket cleanup; observe stream cancellation without delaying the attempt.\n  if (cancellation)\n    void cancellation.then(\n      () => undefined,\n      () => undefined\n    );\n}\n\nfunction sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n  signal?.throwIfAborted();\n  if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));\n  return new Promise((resolve, reject) => {\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const cleanup = () => {\n      if (timer) clearTimeout(timer);\n      signal.removeEventListener('abort', onAbort);\n    };\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    timer = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n    if (signal.aborted) onAbort();\n  });\n}\n\nfunction failedRequest(url: string, phase: ResponsePhase, attempts: number, timeoutMs: number, timedOut: boolean): SenseError {\n  const endpoint = safeUrl(url);\n  if (timedOut) {\n    return new SenseError('EMBED_MODEL', `${endpoint} timed out after ${timeoutMs}ms while waiting for ${phase} (attempt ${attempts}/${MAX_ATTEMPTS}); check the embedding service and network`);\n  }\n  return new SenseError('EMBED_MODEL', `${endpoint} failed while waiting for ${phase} after ${attempts} attempts; check the embedding service and network`);\n}\n\nfunction buffered(response: Response, body: ArrayBuffer, requestUrl: string): BufferedResponse {\n  return {\n    ok: response.ok,\n    status: response.status,\n    headers: response.headers,\n    body,\n    url: safeUrl(response.url || requestUrl),\n  };\n}\n\n// Each attempt covers response headers and the complete successful body. Retry cleanup and\n// backoff honor one caller signal without transferring ownership of that signal.\nexport async function fetchWithRetry(url: string, init: FetchRequestInit, options: FetchRetryOptions = {}): Promise<BufferedResponse> {\n  const timeoutMs = options.attemptTimeoutMs ?? HTTP_ATTEMPT_TIMEOUT_MS;\n  const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    options.signal?.throwIfAborted();\n    const { controller, timeoutReason, cleanup } = attemptController(options.signal, timeoutMs);\n    let response: Response | undefined;\n    let phase: ResponsePhase = 'response headers';\n    let retryDelayMs: number | undefined;\n    try {\n      response = await fetch(url, { ...init, signal: controller.signal });\n      if (!response.ok) {\n        const retryable = response.status === 429 || response.status >= 500;\n        if (retryable && attempt < MAX_ATTEMPTS - 1) {\n          const retryAfter = Number(response.headers.get('retry-after'));\n          retryDelayMs = retryAfter > 0 ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) : baseDelayMs * 2 ** attempt;\n          disposeResponse(response, controller);\n        } else {\n          disposeResponse(response, controller);\n          options.signal?.throwIfAborted();\n          return buffered(response, new ArrayBuffer(0), url);\n        }\n      } else {\n        phase = 'response body';\n        const body = await response.arrayBuffer();\n        options.signal?.throwIfAborted();\n        return buffered(response, body, url);\n      }\n    } catch {\n      try {\n        options.signal?.throwIfAborted();\n      } finally {\n        if (response) disposeResponse(response, controller);\n      }\n      const timedOut = controller.signal.reason === timeoutReason;\n      if (attempt === MAX_ATTEMPTS - 1) throw failedRequest(url, phase, attempt + 1, timeoutMs, timedOut);\n      retryDelayMs = baseDelayMs * 2 ** attempt;\n    } finally {\n      cleanup();\n    }\n    await sleep(retryDelayMs ?? baseDelayMs * 2 ** attempt, options.signal);\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up. Abort it immediately because status does not\n// consume its body, and bound the headers wait for an air-gapped machine.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  const { controller, cleanup } = attemptController(undefined, timeoutMs);\n  try {\n    const response = await fetch(url, { signal: controller.signal });\n    disposeResponse(response, controller);\n    return true;\n  } catch {\n    return false;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["SenseError","MAX_ATTEMPTS","MAX_RETRY_AFTER_MS","DEFAULT_BASE_DELAY_MS","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","safeUrl","url","parsed","URL","username","password","search","hash","href","attemptController","callerSignal","timeoutMs","throwIfAborted","controller","AbortController","timeoutReason","relayAbort","abort","reason","addEventListener","once","aborted","timer","setTimeout","cleanup","clearTimeout","removeEventListener","disposeResponse","response","cancellation","body","bodyUsed","cancel","undefined","then","sleep","ms","signal","Promise","resolve","reject","onAbort","failedRequest","phase","attempts","timedOut","endpoint","buffered","requestUrl","ok","status","headers","fetchWithRetry","init","options","attemptTimeoutMs","baseDelayMs","attempt","retryDelayMs","fetch","retryable","retryAfter","Number","get","Math","min","ArrayBuffer","arrayBuffer","Error","probeReachable"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAE1C,MAAMC,eAAe;AACrB,MAAMC,qBAAqB;AAC3B,MAAMC,wBAAwB;AAE9B,2FAA2F;AAC3F,OAAO,MAAMC,0BAA0B,MAAO;AAC9C,OAAO,MAAMC,gCAAgC,IAAI,MAAO;AAoBxD,SAASC,QAAQC,GAAW;IAC1B,IAAI;QACF,MAAMC,SAAS,IAAIC,IAAIF;QACvBC,OAAOE,QAAQ,GAAG;QAClBF,OAAOG,QAAQ,GAAG;QAClBH,OAAOI,MAAM,GAAG;QAChBJ,OAAOK,IAAI,GAAG;QACd,OAAOL,OAAOM,IAAI;IACpB,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASC,kBACPC,YAAqC,EACrCC,SAAiB;IAMjBD,yBAAAA,mCAAAA,aAAcE,cAAc;IAC5B,MAAMC,aAAa,IAAIC;IACvB,MAAMC,gBAAgB,CAAC;IACvB,MAAMC,aAAa,IAAMH,WAAWI,KAAK,CAACP,yBAAAA,mCAAAA,aAAcQ,MAAM;IAC9DR,yBAAAA,mCAAAA,aAAcS,gBAAgB,CAAC,SAASH,YAAY;QAAEI,MAAM;IAAK;IACjE,IAAIV,yBAAAA,mCAAAA,aAAcW,OAAO,EAAEL;IAC3B,MAAMM,QAAQC,WAAW,IAAMV,WAAWI,KAAK,CAACF,gBAAgBJ;IAChE,OAAO;QACLE;QACAE;QACAS,SAAS;YACPC,aAAaH;YACbZ,yBAAAA,mCAAAA,aAAcgB,mBAAmB,CAAC,SAASV;QAC7C;IACF;AACF;AAEA,SAASW,gBAAgBC,QAAkB,EAAEf,UAA2B;IACtE,MAAMgB,eAAeD,SAASE,IAAI,IAAI,CAACF,SAASG,QAAQ,GAAGH,SAASE,IAAI,CAACE,MAAM,KAAKC;IACpFpB,WAAWI,KAAK;IAChB,gGAAgG;IAChG,IAAIY,cACF,KAAKA,aAAaK,IAAI,CACpB,IAAMD,WACN,IAAMA;AAEZ;AAEA,SAASE,MAAMC,EAAU,EAAEC,MAA+B;IACxDA,mBAAAA,6BAAAA,OAAQzB,cAAc;IACtB,IAAI,CAACyB,QAAQ,OAAO,IAAIC,QAAQ,CAACC,UAAYhB,WAAWgB,SAASH;IACjE,OAAO,IAAIE,QAAQ,CAACC,SAASC;QAC3B,IAAIlB;QACJ,MAAME,UAAU;YACd,IAAIF,OAAOG,aAAaH;YACxBe,OAAOX,mBAAmB,CAAC,SAASe;QACtC;QACA,MAAMA,UAAU;YACdjB;YACAgB,OAAOH,OAAOnB,MAAM;QACtB;QACAmB,OAAOlB,gBAAgB,CAAC,SAASsB,SAAS;YAAErB,MAAM;QAAK;QACvDE,QAAQC,WAAW;YACjBC;YACAe;QACF,GAAGH;QACH,IAAIC,OAAOhB,OAAO,EAAEoB;IACtB;AACF;AAEA,SAASC,cAAczC,GAAW,EAAE0C,KAAoB,EAAEC,QAAgB,EAAEjC,SAAiB,EAAEkC,QAAiB;IAC9G,MAAMC,WAAW9C,QAAQC;IACzB,IAAI4C,UAAU;QACZ,OAAO,IAAInD,WAAW,eAAe,GAAGoD,SAAS,iBAAiB,EAAEnC,UAAU,qBAAqB,EAAEgC,MAAM,UAAU,EAAEC,SAAS,CAAC,EAAEjD,aAAa,0CAA0C,CAAC;IAC7L;IACA,OAAO,IAAID,WAAW,eAAe,GAAGoD,SAAS,0BAA0B,EAAEH,MAAM,OAAO,EAAEC,SAAS,kDAAkD,CAAC;AAC1J;AAEA,SAASG,SAASnB,QAAkB,EAAEE,IAAiB,EAAEkB,UAAkB;IACzE,OAAO;QACLC,IAAIrB,SAASqB,EAAE;QACfC,QAAQtB,SAASsB,MAAM;QACvBC,SAASvB,SAASuB,OAAO;QACzBrB;QACA7B,KAAKD,QAAQ4B,SAAS3B,GAAG,IAAI+C;IAC/B;AACF;AAEA,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,eAAeI,eAAenD,GAAW,EAAEoD,IAAsB,EAAEC,UAA6B,CAAC,CAAC;QACrFA,2BACEA;IADpB,MAAM3C,aAAY2C,4BAAAA,QAAQC,gBAAgB,cAAxBD,uCAAAA,4BAA4BxD;IAC9C,MAAM0D,eAAcF,uBAAAA,QAAQE,WAAW,cAAnBF,kCAAAA,uBAAuBzD;IAC3C,IAAK,IAAI4D,UAAU,GAAGA,UAAU9D,cAAc8D,UAAW;YACvDH;SAAAA,kBAAAA,QAAQjB,MAAM,cAAdiB,sCAAAA,gBAAgB1C,cAAc;QAC9B,MAAM,EAAEC,UAAU,EAAEE,aAAa,EAAES,OAAO,EAAE,GAAGf,kBAAkB6C,QAAQjB,MAAM,EAAE1B;QACjF,IAAIiB;QACJ,IAAIe,QAAuB;QAC3B,IAAIe;QACJ,IAAI;YACF9B,WAAW,MAAM+B,MAAM1D,KAAK;gBAAE,GAAGoD,IAAI;gBAAEhB,QAAQxB,WAAWwB,MAAM;YAAC;YACjE,IAAI,CAACT,SAASqB,EAAE,EAAE;gBAChB,MAAMW,YAAYhC,SAASsB,MAAM,KAAK,OAAOtB,SAASsB,MAAM,IAAI;gBAChE,IAAIU,aAAaH,UAAU9D,eAAe,GAAG;oBAC3C,MAAMkE,aAAaC,OAAOlC,SAASuB,OAAO,CAACY,GAAG,CAAC;oBAC/CL,eAAeG,aAAa,IAAIG,KAAKC,GAAG,CAACJ,aAAa,MAAMjE,sBAAsB4D,cAAc,KAAKC;oBACrG9B,gBAAgBC,UAAUf;gBAC5B,OAAO;wBAELyC;oBADA3B,gBAAgBC,UAAUf;qBAC1ByC,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;oBAC9B,OAAOmC,SAASnB,UAAU,IAAIsC,YAAY,IAAIjE;gBAChD;YACF,OAAO;oBAGLqD;gBAFAX,QAAQ;gBACR,MAAMb,OAAO,MAAMF,SAASuC,WAAW;iBACvCb,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;gBAC9B,OAAOmC,SAASnB,UAAUE,MAAM7B;YAClC;QACF,EAAE,OAAM;YACN,IAAI;oBACFqD;iBAAAA,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;YAChC,SAAU;gBACR,IAAIgB,UAAUD,gBAAgBC,UAAUf;YAC1C;YACA,MAAMgC,WAAWhC,WAAWwB,MAAM,CAACnB,MAAM,KAAKH;YAC9C,IAAI0C,YAAY9D,eAAe,GAAG,MAAM+C,cAAczC,KAAK0C,OAAOc,UAAU,GAAG9C,WAAWkC;YAC1Fa,eAAeF,cAAc,KAAKC;QACpC,SAAU;YACRjC;QACF;QACA,MAAMW,MAAMuB,yBAAAA,0BAAAA,eAAgBF,cAAc,KAAKC,SAASH,QAAQjB,MAAM;IACxE;IACA,MAAM,IAAI+B,MAAM;AAClB;AAEA,2FAA2F;AAC3F,0EAA0E;AAC1E,OAAO,eAAeC,eAAepE,GAAW,EAAEU,YAAY,IAAI;IAChE,MAAM,EAAEE,UAAU,EAAEW,OAAO,EAAE,GAAGf,kBAAkBwB,WAAWtB;IAC7D,IAAI;QACF,MAAMiB,WAAW,MAAM+B,MAAM1D,KAAK;YAAEoC,QAAQxB,WAAWwB,MAAM;QAAC;QAC9DV,gBAAgBC,UAAUf;QAC1B,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT,SAAU;QACRW;IACF;AACF"}

package/dist/esm/embed/openai.js

@@ -1,17 +1,35 @@
 import { SenseError } from '../errors.js';
-import { fetchWithRetry } from './http.js';
+import { effectiveEmbedEndpoint } from './endpoint.js';
+import { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.js';
 const BATCH_CAP = 64;
+function isEmbedding(value) {
+    if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part)=>typeof part === 'number')) return false;
+    return !('index' in value) || value.index === undefined || typeof value.index === 'number';
+}
+function parseBody(body, endpoint) {
+    let decoded;
+    try {
+        decoded = JSON.parse(new TextDecoder().decode(body));
+    } catch  {
+        throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);
+    }
+    if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {
+        throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);
+    }
+    return decoded.data;
+}
 // One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare
 // Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.
 export async function openaiProvider(model, url, keyEnv) {
-    if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
-    const base = url.replace(/\/+$/, '');
+    const base = effectiveEmbedEndpoint('openai', url);
+    if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
     const headers = {
         'content-type': 'application/json'
     };
     const key = keyEnv ? process.env[keyEnv] : undefined;
     if (key) headers.authorization = `Bearer ${key}`;
-    async function post(texts) {
+    async function post(texts, options) {
+        var _options_signal;
         const res = await fetchWithRetry(`${base}/embeddings`, {
             method: 'POST',
             headers,
@@ -19,17 +37,21 @@ export async function openaiProvider(model, url, keyEnv) {
                 model,
                 input: texts
             })
+        }, {
+            signal: options === null || options === void 0 ? void 0 : options.signal,
+            attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
         });
-        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);
-        const body = await res.json();
+        options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);
+        const body = parseBody(res.body, res.url);
         // The schema carries `index` because response order is not promised, and the caller maps
         // vectors back to chunks positionally: sort by it whenever every element has one.
-        const data = body.data.every((d)=>typeof d.index === 'number') ? [
-            ...body.data
-        ].sort((a, b)=>a.index - b.index) : body.data;
+        const data = body.every((d)=>typeof d.index === 'number') ? [
+            ...body
+        ].sort((a, b)=>a.index - b.index) : body;
         // A server that drops an input (empty or over-length strings are the usual cause) would
         // otherwise shift every later vector onto the wrong chunk, silently.
-        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);
+        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);
         return data.map((d)=>Float32Array.from(d.embedding));
     }
     const dims = (await post([
@@ -42,8 +64,8 @@ export async function openaiProvider(model, url, keyEnv) {
         dims,
         batchCap: BATCH_CAP,
         embedDocuments: post,
-        embedQuery: async (text)=>(await post([
+        embedQuery: async (text, options)=>(await post([
                 text
-            ]))[0]
+            ], options))[0]
     };
 }

package/dist/esm/embed/openai.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","openaiProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","res","method","body","JSON","stringify","input","ok","status","json","data","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,IAAI,CAACD,KAAK,MAAM,IAAIL,WAAW,eAAe;IAC9C,MAAMO,OAAOF,IAAIG,OAAO,CAAC,QAAQ;IACjC,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe;QACjC,MAAMC,MAAM,MAAMhB,eAAe,GAAGM,KAAK,WAAW,CAAC,EAAE;YAAEW,QAAQ;YAAQT;YAASU,MAAMC,KAAKC,SAAS,CAAC;gBAAEjB;gBAAOkB,OAAON;YAAM;QAAG;QAChI,IAAI,CAACC,IAAIM,EAAE,EAAE,MAAM,IAAIvB,WAAW,eAAe,GAAGO,KAAK,oBAAoB,EAAEU,IAAIO,MAAM,EAAE;QAC3F,MAAML,OAAQ,MAAMF,IAAIQ,IAAI;QAC5B,yFAAyF;QACzF,kFAAkF;QAClF,MAAMC,OAAOP,KAAKO,IAAI,CAACC,KAAK,CAAC,CAACC,IAAM,OAAOA,EAAEC,KAAK,KAAK,YAAY;eAAIV,KAAKO,IAAI;SAAC,CAACI,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK,IAAeV,KAAKO,IAAI;QACvJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIA,KAAKO,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAIjC,WAAW,eAAe,GAAGO,KAAK,qBAAqB,EAAEmB,KAAKO,MAAM,CAAC,gBAAgB,EAAEjB,MAAMiB,MAAM,CAAC,OAAO,CAAC;QACxJ,OAAOP,KAAKQ,GAAG,CAAC,CAACN,IAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;IACtD;IAEA,MAAMC,OAAO,AAAC,CAAA,MAAMvB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACkB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEH,OAAO;QAAEkC;QAAME,UAAUtC;QAAWuC,gBAAgB1B;QAAM2B,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM5B,KAAK;gBAAC4B;aAAK,CAAA,CAAE,CAAC,EAAE;IAAC;AAC/I"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\ninterface OpenaiEmbedding {\n  embedding: number[];\n  index?: number;\n}\n\nfunction isEmbedding(value: unknown): value is OpenaiEmbedding {\n  if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part) => typeof part === 'number')) return false;\n  return !('index' in value) || value.index === undefined || typeof value.index === 'number';\n}\n\nfunction parseBody(body: ArrayBuffer, endpoint: string): OpenaiEmbedding[] {\n  let decoded: unknown;\n  try {\n    decoded = JSON.parse(new TextDecoder().decode(body));\n  } catch {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);\n  }\n  if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);\n  }\n  return decoded.data;\n}\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = effectiveEmbedEndpoint('openai', url);\n  if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) }, { signal: options?.signal, attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n    options?.signal?.throwIfAborted();\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n    const body = parseBody(res.body, res.url);\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.every((d): d is OpenaiEmbedding & { index: number } => typeof d.index === 'number') ? [...body].sort((a, b) => a.index - b.index) : body;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text, options) => (await post([text], options))[0] };\n}\n"],"names":["SenseError","effectiveEmbedEndpoint","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","BATCH_CAP","isEmbedding","value","Array","isArray","embedding","every","part","index","undefined","parseBody","body","endpoint","decoded","JSON","parse","TextDecoder","decode","data","openaiProvider","model","url","keyEnv","base","headers","key","process","env","authorization","post","texts","options","res","method","stringify","input","signal","attemptTimeoutMs","throwIfAborted","ok","status","d","sort","a","b","length","map","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,EAAEC,uBAAuB,QAAQ,YAAY;AAGpE,MAAMC,YAAY;AAOlB,SAASC,YAAYC,KAAc;IACjC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,YAAY,CAAE,CAAA,eAAeA,KAAI,KAAM,CAACC,MAAMC,OAAO,CAACF,MAAMG,SAAS,KAAK,CAACH,MAAMG,SAAS,CAACC,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS,WAAW,OAAO;IACpL,OAAO,CAAE,CAAA,WAAWL,KAAI,KAAMA,MAAMM,KAAK,KAAKC,aAAa,OAAOP,MAAMM,KAAK,KAAK;AACpF;AAEA,SAASE,UAAUC,IAAiB,EAAEC,QAAgB;IACpD,IAAIC;IACJ,IAAI;QACFA,UAAUC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAACN;IAChD,EAAE,OAAM;QACN,MAAM,IAAIf,WAAW,eAAe,GAAGgB,SAAS,sBAAsB,CAAC;IACzE;IACA,IAAIC,YAAY,QAAQ,OAAOA,YAAY,YAAY,CAAE,CAAA,UAAUA,OAAM,KAAM,CAACV,MAAMC,OAAO,CAACS,QAAQK,IAAI,KAAK,CAACL,QAAQK,IAAI,CAACZ,KAAK,CAACL,cAAc;QAC/I,MAAM,IAAIL,WAAW,eAAe,GAAGgB,SAAS,uCAAuC,CAAC;IAC1F;IACA,OAAOC,QAAQK,IAAI;AACrB;AAEA,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO1B,uBAAuB,UAAUwB;IAC9C,IAAI,CAACE,MAAM,MAAM,IAAI3B,WAAW,eAAe;IAC/C,MAAM4B,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGb;IAC3C,IAAIgB,KAAKD,QAAQI,aAAa,GAAG,CAAC,OAAO,EAAEH,KAAK;IAEhD,eAAeI,KAAKC,KAAe,EAAEC,OAA0B;YAE7DA;QADA,MAAMC,MAAM,MAAMlC,eAAe,GAAGyB,KAAK,WAAW,CAAC,EAAE;YAAEU,QAAQ;YAAQT;YAASb,MAAMG,KAAKoB,SAAS,CAAC;gBAAEd;gBAAOe,OAAOL;YAAM;QAAG,GAAG;YAAEM,MAAM,EAAEL,oBAAAA,8BAAAA,QAASK,MAAM;YAAEC,kBAAkBtC;QAAwB;QACxMgC,oBAAAA,+BAAAA,kBAAAA,QAASK,MAAM,cAAfL,sCAAAA,gBAAiBO,cAAc;QAC/B,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAI3C,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,SAAS,EAAEW,IAAIQ,MAAM,EAAE;QACnF,MAAM7B,OAAOD,UAAUsB,IAAIrB,IAAI,EAAEqB,IAAIX,GAAG;QACxC,yFAAyF;QACzF,kFAAkF;QAClF,MAAMH,OAAOP,KAAKL,KAAK,CAAC,CAACmC,IAAgD,OAAOA,EAAEjC,KAAK,KAAK,YAAY;eAAIG;SAAK,CAAC+B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEnC,KAAK,GAAGoC,EAAEpC,KAAK,IAAIG;QACtJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIO,KAAK2B,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAIjD,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,UAAU,EAAEH,KAAK2B,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAChJ,OAAO3B,KAAK4B,GAAG,CAAC,CAACL,IAAMM,aAAaC,IAAI,CAACP,EAAEpC,SAAS;IACtD;IAEA,MAAM4C,OAAO,AAAC,CAAA,MAAMpB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEK,IAAI,CAAC,OAAO,EAAE3B,KAAK,CAAC,EAAEH,OAAO;QAAE6B;QAAME,UAAUnD;QAAWoD,gBAAgBvB;QAAMwB,YAAY,OAAOC,MAAMvB,UAAY,AAAC,CAAA,MAAMF,KAAK;gBAACyB;aAAK,EAAEvB,QAAO,CAAE,CAAC,EAAE;IAAC;AACjK"}

package/dist/esm/embed/query.d.ts

@@ -1,12 +1,18 @@
-import type { Config } from '../config/index.js';
+import { type Config } from '../config/index.js';
 import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
+import { type EmbedCallOptions } from './types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options?: EmbedCallOptions): Promise<void>;
 export declare function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void>;
-export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export interface PreparedSemanticQuery {
+    readonly vector: Float32Array;
+    readonly storeDims: number;
+}
+export declare function prepareSemanticQuery(cfg: Config, terms: string, options?: EmbedCallOptions): Promise<PreparedSemanticQuery>;
+export declare function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
 export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;

package/dist/esm/embed/query.js

@@ -1,8 +1,13 @@
+import { featureSignature } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { embed } from '../features/embed.js';
+import { FEATURES } from '../features/index.js';
 import { progress } from '../output/progress.js';
 import { parseSource } from '../scan/index.js';
-import { takeChunkText } from './handoff.js';
+import { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.js';
+import { getMeta } from '../store/shared.js';
+import { embeddingSignatureCompatible } from '../store/signature.js';
+import { identifyChunkText, takeChunkText } from './handoff.js';
 import { checkLanguageFit } from './langfit.js';
 import { getProvider } from './registry.js';
 import { STORE_DIMS } from './types.js';
@@ -28,38 +33,64 @@ export function toStore(full, dims, int8) {
 // Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in
 // this process (handoff.ts); a row left pending by an earlier command re-derives from the exact
 // source string persisted with that indexed generation.
-export async function embedPending(store, cfg, allowed) {
+function staleEmbeddingWork(reason) {
+    return new SenseError('INDEX_NOT_READY', `${reason}; run \`sense build\` (or library build(config)) again to prepare the current index generation`);
+}
+// expectedSignature is captured by open() before provider work. Direct internal callers omit it
+// and capture the durable value here, before selecting chunks or constructing a provider.
+export async function embedPending(store, cfg, allowed, expectedSignature, options = {}) {
     var _takeChunkText;
-    const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table
-    const dirty = (await store.vectors.pending()).filter((row)=>{
+    var _options_signal, _options_signal1, _options_signal2, _options_signal3, // Over the exact texts about to be embedded, before any of them are: a mismatch fails
+    // this run loudly instead of quietly storing vectors from the wrong model.
+    _options_signal4, _options_signal5;
+    (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+    const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;
+    (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+    if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {
+        throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');
+    }
+    const pendingStmt = await store.prepare('SELECT "path", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
+    const dirty = (await pendingStmt.all()).filter((row)=>{
         var _ref;
         return (_ref = allowed === null || allowed === void 0 ? void 0 : allowed.has(row.path)) !== null && _ref !== void 0 ? _ref : true;
     });
+    (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
     if (dirty.length === 0) return;
+    const provider = await getProvider(cfg, options);
+    (_options_signal3 = options.signal) === null || _options_signal3 === void 0 ? void 0 : _options_signal3.throwIfAborted();
+    const submittedSignature = featureSignature(cfg, FEATURES);
+    if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {
+        throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');
+    }
     const storeDims = Math.min(STORE_DIMS, provider.dims);
     const byPath = new Map();
     for (const row of dirty){
         var _byPath_get;
         const list = (_byPath_get = byPath.get(row.path)) !== null && _byPath_get !== void 0 ? _byPath_get : [];
-        list.push(row.chunk);
+        list.push(row);
         byPath.set(row.path, list);
     }
     const textByPath = (_takeChunkText = takeChunkText(store)) !== null && _takeChunkText !== void 0 ? _takeChunkText : new Map();
     const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE "path" = ?');
     const jobs = [];
-    for (const [path, chunkIdxs] of byPath){
+    for (const [path, pending] of byPath){
+        var _options_signal6;
         const texts = textByPath.get(path);
-        if (texts && chunkIdxs.every((idx)=>idx < texts.length)) {
-            for (const idx of chunkIdxs)jobs.push({
+        if (texts && pending.every((row)=>{
+            var _texts_row_chunk;
+            return ((_texts_row_chunk = texts[row.chunk]) === null || _texts_row_chunk === void 0 ? void 0 : _texts_row_chunk.identity) === row.content_identity;
+        })) {
+            for (const row of pending)jobs.push({
                 path,
-                chunk: idx,
-                text: texts[idx]
+                chunk: row.chunk,
+                ...texts[row.chunk]
             });
             continue;
         }
         // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive
         // from that indexed generation's stored source, never from a newer live file.
         const source = await sourceStmt.get(path);
+        (_options_signal6 = options.signal) === null || _options_signal6 === void 0 ? void 0 : _options_signal6.throwIfAborted();
         if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \`sense build --force\` (or library build(config, { force: true })) before preparing vectors`);
         const chunks = parseSource({
             relPath: path,
@@ -72,26 +103,33 @@ export async function embedPending(store, cfg, allowed) {
         }, source.text, [
             embed
         ], cfg).doc.extracted.embed;
-        for (const idx of chunkIdxs){
-            const chunk = chunks[idx];
+        for (const row of pending){
+            const chunk = chunks[row.chunk];
             if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \`sense build --force\` (or library build(config, { force: true }))`);
+            const identified = identifyChunkText(chunk.text);
+            if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);
             jobs.push({
                 path,
-                chunk: idx,
-                text: chunk.text
+                chunk: row.chunk,
+                ...identified
             });
         }
     }
-    // Over the exact texts about to be embedded, before any of them are: a mismatch fails
-    // this run loudly instead of quietly storing vectors from the wrong model.
+    (_options_signal4 = options.signal) === null || _options_signal4 === void 0 ? void 0 : _options_signal4.throwIfAborted();
     await checkLanguageFit(store, provider, jobs.map((j)=>j.text));
+    (_options_signal5 = options.signal) === null || _options_signal5 === void 0 ? void 0 : _options_signal5.throwIfAborted();
     // Preparing a large configured scope can take long enough to look stalled, so report each
     // completed provider batch.
     const report = progress('embedding chunks', jobs.length);
     for(let i = 0; i < jobs.length; i += provider.batchCap){
+        var _options_signal7, _options_signal8, _options_signal9;
         const batch = jobs.slice(i, i + provider.batchCap);
-        const vectors = await provider.embedDocuments(batch.map((j)=>j.text));
-        await store.vectors.writeVectors(batch.map((job, j)=>{
+        (_options_signal7 = options.signal) === null || _options_signal7 === void 0 ? void 0 : _options_signal7.throwIfAborted();
+        if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');
+        const vectors = await provider.embedDocuments(batch.map((j)=>j.text), options);
+        (_options_signal8 = options.signal) === null || _options_signal8 === void 0 ? void 0 : _options_signal8.throwIfAborted();
+        if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');
+        const writes = batch.map((job, j)=>{
             const { v, scale } = toStore(vectors[j], storeDims, true);
             const q = new Int8Array(storeDims);
             for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
@@ -101,7 +139,28 @@ export async function embedPending(store, cfg, allowed) {
                 scale,
                 vector: Buffer.from(q.buffer)
             };
-        }));
+        });
+        (_options_signal9 = options.signal) === null || _options_signal9 === void 0 ? void 0 : _options_signal9.throwIfAborted();
+        await store.transaction(async ()=>{
+            var // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws
+            // inside this transaction and rolls every row in the provider batch back together.
+            _options_signal, _options_signal1, _options_signal2;
+            (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            await store.vectors.writeVectors(writes);
+            (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+            if (await getMeta(store, FEATURE_SIGNATURE_META_KEY) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');
+            const currentStmt = await store.prepare(`SELECT "path", chunk, content_identity FROM embeddings WHERE ${batch.map(()=>'("path" = ? AND chunk = ?)').join(' OR ')}`);
+            const current = await currentStmt.all(...batch.flatMap((job)=>[
+                    job.path,
+                    job.chunk
+                ]));
+            const identities = new Map(current.map((row)=>[
+                    `${row.path}\0${Number(row.chunk)}`,
+                    row.content_identity
+                ]));
+            if (batch.some((job)=>identities.get(`${job.path}\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');
+            (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
+        });
         report.tick(Math.min(i + provider.batchCap, jobs.length));
     }
     report.finish();
@@ -112,22 +171,35 @@ export async function ensureDocumentEmbeddings(store, _cfg, allowed) {
         throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \`sense build\` (or library build(config)) before querying with build disabled`);
     }
 }
-// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as
-// lexical syntax. Similarity comes back because the fused score cannot express match quality.
-export async function semanticCandidates(store, cfg, terms, fetch, allowed) {
+// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as
+// lexical syntax before the query is embedded, matching the prior semantic query behavior.
+export async function prepareSemanticQuery(cfg, terms, options = {}) {
     var _cfg_rootDir, _terms_match;
+    var _options_signal, _options_signal1, _options_signal2;
     const rootDir = (_cfg_rootDir = cfg.rootDir) !== null && _cfg_rootDir !== void 0 ? _cfg_rootDir : cfg.baseDir;
     if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');
-    const provider = await getProvider(cfg);
-    const storeDims = Math.min(STORE_DIMS, provider.dims);
     const text = ((_terms_match = terms.match(/[\p{L}\p{N}]+/gu)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>![
             'AND',
             'OR',
             'NOT',
             'NEAR'
         ].includes(t)).join(' ');
-    const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);
-    return store.vectors.candidates(qv, storeDims, fetch, allowed);
+    (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+    const provider = await getProvider(cfg, options);
+    (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const queryVector = await provider.embedQuery(text, options);
+    (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
+    const { v: qv } = toStore(queryVector, storeDims, false);
+    return {
+        vector: qv,
+        storeDims
+    };
+}
+// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot
+// and the prepared vector, so this performs no provider work and only delegates to the store.
+export function semanticCandidates(store, prepared, fetch, allowed) {
+    return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);
 }
 // Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
 // "this note has no text", which look the same in an empty result.

package/dist/esm/embed/query.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import type { Config, ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void> {\n  const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table\n  const dirty = (await store.vectors.pending()).filter((row) => allowed?.has(row.path) ?? true);\n  if (dirty.length === 0) return;\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, number[]>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row.chunk);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map<string, string[]>();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string }> = [];\n  for (const [path, chunkIdxs] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && chunkIdxs.every((idx) => idx < texts.length)) {\n      for (const idx of chunkIdxs) jobs.push({ path, chunk: idx, text: texts[idx] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const idx of chunkIdxs) {\n      const chunk = chunks[idx];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      jobs.push({ path, chunk: idx, text: chunk.text });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    const vectors = await provider.embedDocuments(batch.map((j) => j.text));\n    await store.vectors.writeVectors(\n      batch.map((job, j) => {\n        const { v, scale } = toStore(vectors[j], storeDims, true);\n        const q = new Int8Array(storeDims);\n        for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n        return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n      })\n    );\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\n// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as\n// lexical syntax. Similarity comes back because the fused score cannot express match quality.\nexport async function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const provider = await getProvider(cfg);\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);\n\n  return store.vectors.candidates(qv, storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["SenseError","embed","progress","parseSource","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","embedPending","store","cfg","allowed","provider","dirty","vectors","pending","filter","row","has","path","length","storeDims","min","byPath","Map","list","get","push","chunk","set","textByPath","sourceStmt","prepare","jobs","chunkIdxs","texts","every","idx","text","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","presets","doc","extracted","map","j","report","i","batchCap","batch","slice","embedDocuments","writeVectors","job","q","Int8Array","round","vector","from","buffer","tick","finish","ensureDocumentEmbeddings","_cfg","semanticCandidates","terms","fetch","rootDir","baseDir","match","t","includes","join","qv","embedQuery","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AACA,SAASA,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAE/C,SAASC,aAAa,QAAQ,eAAe;AAC7C,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAASC,UAAU,QAAQ,aAAa;AAExC,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,OAAO,eAAeE,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B;QAatEpB;IAZnB,MAAMqB,WAAW,MAAMnB,YAAYiB,MAAM,kDAAkD;IAC3F,MAAMG,QAAQ,AAAC,CAAA,MAAMJ,MAAMK,OAAO,CAACC,OAAO,EAAC,EAAGC,MAAM,CAAC,CAACC;;uBAAQN,oBAAAA,8BAAAA,QAASO,GAAG,CAACD,IAAIE,IAAI,wCAAK;;IACxF,IAAIN,MAAMO,MAAM,KAAK,GAAG;IACxB,MAAMC,YAAYlB,KAAKmB,GAAG,CAAC5B,YAAYkB,SAASf,IAAI;IAEpD,MAAM0B,SAAS,IAAIC;IACnB,KAAK,MAAMP,OAAOJ,MAAO;YACVU;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACT,IAAIE,IAAI,eAAnBI,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACV,IAAIW,KAAK;QACnBL,OAAOM,GAAG,CAACZ,IAAIE,IAAI,EAAEM;IACvB;IAEA,MAAMK,cAAavC,iBAAAA,cAAckB,oBAAdlB,4BAAAA,iBAAwB,IAAIiC;IAC/C,MAAMO,aAAa,MAAMtB,MAAMuB,OAAO,CAAC;IAEvC,MAAMC,OAA6D,EAAE;IACrE,KAAK,MAAM,CAACd,MAAMe,UAAU,IAAIX,OAAQ;QACtC,MAAMY,QAAQL,WAAWJ,GAAG,CAACP;QAC7B,IAAIgB,SAASD,UAAUE,KAAK,CAAC,CAACC,MAAQA,MAAMF,MAAMf,MAAM,GAAG;YACzD,KAAK,MAAMiB,OAAOH,UAAWD,KAAKN,IAAI,CAAC;gBAAER;gBAAMS,OAAOS;gBAAKC,MAAMH,KAAK,CAACE,IAAI;YAAC;YAC5E;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAME,SAAU,MAAMR,WAAWL,GAAG,CAACP;QACrC,IAAI,CAACoB,QAAQ,MAAM,IAAIpD,WAAW,mBAAmB,CAAC,mCAAmC,EAAEgC,KAAK,kGAAkG,CAAC;QACnM,MAAMqB,SAASlD,YAAY;YAAEmD,SAAStB;YAAMuB,SAASvB;YAAMwB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOD,IAAI;YAAGU,SAAS,EAAE;YAAE5D,OAAO;QAAK,GAAGmD,OAAOD,IAAI,EAAE;YAAClD;SAAM,EAAEsB,KAAKuC,GAAG,CAACC,SAAS,CAAC9D,KAAK;QACnM,KAAK,MAAMiD,OAAOH,UAAW;YAC3B,MAAMN,QAAQY,MAAM,CAACH,IAAI;YACzB,IAAI,CAACT,OAAO,MAAM,IAAIzC,WAAW,mBAAmB,CAAC,4BAA4B,EAAEgC,KAAK,2GAA2G,CAAC;YACpMc,KAAKN,IAAI,CAAC;gBAAER;gBAAMS,OAAOS;gBAAKC,MAAMV,MAAMU,IAAI;YAAC;QACjD;IACF;IAEA,sFAAsF;IACtF,2EAA2E;IAC3E,MAAM9C,iBACJiB,OACAG,UACAqB,KAAKkB,GAAG,CAAC,CAACC,IAAMA,EAAEd,IAAI;IAGxB,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAMe,SAAShE,SAAS,oBAAoB4C,KAAKb,MAAM;IACvD,IAAK,IAAIkC,IAAI,GAAGA,IAAIrB,KAAKb,MAAM,EAAEkC,KAAK1C,SAAS2C,QAAQ,CAAE;QACvD,MAAMC,QAAQvB,KAAKwB,KAAK,CAACH,GAAGA,IAAI1C,SAAS2C,QAAQ;QACjD,MAAMzC,UAAU,MAAMF,SAAS8C,cAAc,CAACF,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEd,IAAI;QACrE,MAAM7B,MAAMK,OAAO,CAAC6C,YAAY,CAC9BH,MAAML,GAAG,CAAC,CAACS,KAAKR;YACd,MAAM,EAAErD,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQmB,OAAO,CAACsC,EAAE,EAAE/B,WAAW;YACpD,MAAMwC,IAAI,IAAIC,UAAUzC;YACxB,IAAK,IAAInB,IAAI,GAAGA,IAAImB,WAAWnB,IAAK2D,CAAC,CAAC3D,EAAE,GAAGC,KAAK4D,KAAK,CAAChE,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEc,MAAMyC,IAAIzC,IAAI;gBAAES,OAAOgC,IAAIhC,KAAK;gBAAEvB;gBAAO2D,QAAQlB,OAAOmB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;QAEFb,OAAOc,IAAI,CAAChE,KAAKmB,GAAG,CAACgC,IAAI1C,SAAS2C,QAAQ,EAAEtB,KAAKb,MAAM;IACzD;IACAiC,OAAOe,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB5D,KAAY,EAAE6D,IAAY,EAAE3D,OAA4B;IACrG,MAAMI,UAAU,AAAC,CAAA,MAAMN,MAAMK,OAAO,CAACC,OAAO,EAAC,EAAGC,MAAM,CAAC,CAACC,MAAQN,QAAQO,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIJ,QAAQK,MAAM,GAAG,GAAG;QACtB,MAAM,IAAIjC,WAAW,mBAAmB,GAAG4B,QAAQK,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAEA,6FAA6F;AAC7F,8FAA8F;AAC9F,OAAO,eAAemD,mBAAmB9D,KAAY,EAAEC,GAAW,EAAE8D,KAAa,EAAEC,KAAa,EAAE9D,OAAqB;QACrG,cAIF6D;IAJd,MAAME,WAAU,eAAA,AAAChE,IAAgCgE,OAAO,cAAxC,0BAAA,eAA4C,AAAChE,IAAgCiE,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIvF,WAAW,eAAe;IAClD,MAAMyB,WAAW,MAAMnB,YAAYiB;IACnC,MAAMW,YAAYlB,KAAKmB,GAAG,CAAC5B,YAAYkB,SAASf,IAAI;IACpD,MAAMyC,OAAO,EAACkC,eAAAA,MAAMI,KAAK,CAAC,gCAAZJ,0BAAAA,eAAkC,EAAE,EAAExD,MAAM,CAAC,CAAC6D,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIE,IAAI,CAAC;IAClH,MAAM,EAAEhF,GAAGiF,EAAE,EAAE,GAAGrF,QAAQ,MAAMiB,SAASqE,UAAU,CAAC3C,OAAOjB,WAAW;IAEtE,OAAOZ,MAAMK,OAAO,CAACoE,UAAU,CAACF,IAAI3D,WAAWoD,OAAO9D;AACxD;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAewE,aAAa1E,KAAY,EAAEU,IAAY;IAC3D,OAAOV,MAAMK,OAAO,CAACsE,SAAS,CAACjE;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAekE,aAAa5E,KAAY,EAAE6D,IAAY,EAAEnD,IAAY,EAAEmE,IAAgE;IAC3I,OAAO7E,MAAMK,OAAO,CAACyE,OAAO,CAACpE,MAAMmE;AACrC"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/query.ts"],"sourcesContent":["import { type Config, featureSignature, type ResolvedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport type { Chunk } from '../features/embed.ts';\nimport { embed } from '../features/embed.ts';\nimport { FEATURES } from '../features/index.ts';\nimport { progress } from '../output/progress.ts';\nimport { parseSource } from '../scan/index.ts';\nimport { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.ts';\nimport { getMeta } from '../store/shared.ts';\nimport { embeddingSignatureCompatible } from '../store/signature.ts';\nimport type { Store, VectorCandidate, VectorSimilar } from '../store/types.ts';\nimport { identifyChunkText, takeChunkText } from './handoff.ts';\nimport { checkLanguageFit } from './langfit.ts';\nimport { getProvider } from './registry.ts';\nimport { type EmbedCallOptions, STORE_DIMS } from './types.ts';\n\n// Slice + re-normalize (Matryoshka); optionally round through int8 storage. Exported for\n// benchmark/lib/embed.mjs, which scores the same lever math offline.\nexport function toStore(full: Float32Array, dims: number, int8: boolean): { v: Float32Array; scale: number } {\n  const v = new Float32Array(dims);\n  let norm = 0;\n  for (let d = 0; d < dims; d++) norm += full[d] * full[d];\n  norm = Math.sqrt(norm) + 1e-32;\n  for (let d = 0; d < dims; d++) v[d] = full[d] / norm;\n  if (!int8) return { v, scale: 1 };\n  let max = 0;\n  for (let d = 0; d < dims; d++) max = Math.max(max, Math.abs(v[d]));\n  return { v, scale: max / 127 || 1 };\n}\n\n// Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in\n// this process (handoff.ts); a row left pending by an earlier command re-derives from the exact\n// source string persisted with that indexed generation.\nfunction staleEmbeddingWork(reason: string): SenseError {\n  return new SenseError('INDEX_NOT_READY', `${reason}; run \\`sense build\\` (or library build(config)) again to prepare the current index generation`);\n}\n\n// expectedSignature is captured by open() before provider work. Direct internal callers omit it\n// and capture the durable value here, before selecting chunks or constructing a provider.\nexport async function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options: EmbedCallOptions = {}): Promise<void> {\n  options.signal?.throwIfAborted();\n  const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;\n  options.signal?.throwIfAborted();\n  if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {\n    throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');\n  }\n\n  const pendingStmt = await store.prepare('SELECT \"path\", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY \"path\", chunk');\n  const dirty = ((await pendingStmt.all()) as Array<{ path: string; chunk: number; content_identity: string }>).filter((row) => allowed?.has(row.path) ?? true);\n  options.signal?.throwIfAborted();\n  if (dirty.length === 0) return;\n\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const submittedSignature = featureSignature(cfg, FEATURES);\n  if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {\n    throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');\n  }\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n\n  const byPath = new Map<string, typeof dirty>();\n  for (const row of dirty) {\n    const list = byPath.get(row.path) ?? [];\n    list.push(row);\n    byPath.set(row.path, list);\n  }\n\n  const textByPath = takeChunkText(store) ?? new Map();\n  const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE \"path\" = ?');\n\n  const jobs: Array<{ path: string; chunk: number; text: string; identity: string }> = [];\n  for (const [path, pending] of byPath) {\n    const texts = textByPath.get(path);\n    if (texts && pending.every((row) => texts[row.chunk]?.identity === row.content_identity)) {\n      for (const row of pending) jobs.push({ path, chunk: row.chunk, ...texts[row.chunk] });\n      continue;\n    }\n    // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive\n    // from that indexed generation's stored source, never from a newer live file.\n    const source = (await sourceStmt.get(path)) as { text: string } | undefined;\n    options.signal?.throwIfAborted();\n    if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \\`sense build --force\\` (or library build(config, { force: true })) before preparing vectors`);\n    const chunks = parseSource({ relPath: path, absPath: path, mtimeMs: 0, ctimeMs: 0, size: Buffer.byteLength(source.text), presets: [], embed: true }, source.text, [embed], cfg).doc.extracted.embed as Chunk[];\n    for (const row of pending) {\n      const chunk = chunks[row.chunk];\n      if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \\`sense build --force\\` (or library build(config, { force: true }))`);\n      const identified = identifyChunkText(chunk.text);\n      if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);\n      jobs.push({ path, chunk: row.chunk, ...identified });\n    }\n  }\n\n  // Over the exact texts about to be embedded, before any of them are: a mismatch fails\n  // this run loudly instead of quietly storing vectors from the wrong model.\n  options.signal?.throwIfAborted();\n  await checkLanguageFit(\n    store,\n    provider,\n    jobs.map((j) => j.text)\n  );\n  options.signal?.throwIfAborted();\n\n  // Preparing a large configured scope can take long enough to look stalled, so report each\n  // completed provider batch.\n  const report = progress('embedding chunks', jobs.length);\n  for (let i = 0; i < jobs.length; i += provider.batchCap) {\n    const batch = jobs.slice(i, i + provider.batchCap);\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');\n    const vectors = await provider.embedDocuments(\n      batch.map((j) => j.text),\n      options\n    );\n    options.signal?.throwIfAborted();\n    if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');\n    const writes = batch.map((job, j) => {\n      const { v, scale } = toStore(vectors[j], storeDims, true);\n      const q = new Int8Array(storeDims);\n      for (let d = 0; d < storeDims; d++) q[d] = Math.round(v[d] / scale);\n      return { path: job.path, chunk: job.chunk, scale, vector: Buffer.from(q.buffer) };\n    });\n    options.signal?.throwIfAborted();\n    await store.transaction(async () => {\n      // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws\n      // inside this transaction and rolls every row in the provider batch back together.\n      options.signal?.throwIfAborted();\n      await store.vectors.writeVectors(writes);\n      options.signal?.throwIfAborted();\n      if ((await getMeta(store, FEATURE_SIGNATURE_META_KEY)) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');\n      const currentStmt = await store.prepare(`SELECT \"path\", chunk, content_identity FROM embeddings WHERE ${batch.map(() => '(\"path\" = ? AND chunk = ?)').join(' OR ')}`);\n      const current = (await currentStmt.all(...batch.flatMap((job) => [job.path, job.chunk]))) as Array<{ path: string; chunk: number; content_identity: string }>;\n      const identities = new Map(current.map((row) => [`${row.path}\\0${Number(row.chunk)}`, row.content_identity]));\n      if (batch.some((job) => identities.get(`${job.path}\\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');\n      options.signal?.throwIfAborted();\n    });\n    report.tick(Math.min(i + provider.batchCap, jobs.length));\n  }\n  report.finish();\n}\n\nexport async function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void> {\n  const pending = (await store.vectors.pending()).filter((row) => allowed.has(row.path));\n  if (pending.length > 0) {\n    throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \\`sense build\\` (or library build(config)) before querying with build disabled`);\n  }\n}\n\nexport interface PreparedSemanticQuery {\n  readonly vector: Float32Array;\n  readonly storeDims: number;\n}\n\n// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as\n// lexical syntax before the query is embedded, matching the prior semantic query behavior.\nexport async function prepareSemanticQuery(cfg: Config, terms: string, options: EmbedCallOptions = {}): Promise<PreparedSemanticQuery> {\n  const rootDir = (cfg as Partial<ResolvedConfig>).rootDir ?? (cfg as Partial<ResolvedConfig>).baseDir;\n  if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');\n  const text = (terms.match(/[\\p{L}\\p{N}]+/gu) ?? []).filter((t) => !['AND', 'OR', 'NOT', 'NEAR'].includes(t)).join(' ');\n  options.signal?.throwIfAborted();\n  const provider = await getProvider(cfg, options);\n  options.signal?.throwIfAborted();\n  const storeDims = Math.min(STORE_DIMS, provider.dims);\n  const queryVector = await provider.embedQuery(text, options);\n  options.signal?.throwIfAborted();\n  const { v: qv } = toStore(queryVector, storeDims, false);\n  return { vector: qv, storeDims };\n}\n\n// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot\n// and the prepared vector, so this performs no provider work and only delegates to the store.\nexport function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]> {\n  return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);\n}\n\n// Whether a note has any chunk with a vector. Distinguishes \"nothing is near this note\" from\n// \"this note has no text\", which look the same in an empty result.\nexport async function hasEmbedding(store: Store, path: string): Promise<boolean> {\n  return store.vectors.hasVector(path);\n}\n\n// Note-to-note similarity is the max cosine over (target chunk, other chunk) pairs, one linear\n// scan of stored vectors.\nexport async function similarNotes(store: Store, _cfg: Config, path: string, opts: { exclude: Set<string>; allowed?: Set<string>; k: number }): Promise<VectorSimilar[]> {\n  return store.vectors.similar(path, opts);\n}\n"],"names":["featureSignature","SenseError","embed","FEATURES","progress","parseSource","FEATURE_SIGNATURE_META_KEY","getMeta","embeddingSignatureCompatible","identifyChunkText","takeChunkText","checkLanguageFit","getProvider","STORE_DIMS","toStore","full","dims","int8","v","Float32Array","norm","d","Math","sqrt","scale","max","abs","staleEmbeddingWork","reason","embedPending","store","cfg","allowed","expectedSignature","options","signal","throwIfAborted","expectedDurableSignature","undefined","pendingStmt","prepare","dirty","all","filter","row","has","path","length","provider","submittedSignature","storeDims","min","byPath","Map","list","get","push","set","textByPath","sourceStmt","jobs","pending","texts","every","chunk","identity","content_identity","source","chunks","relPath","absPath","mtimeMs","ctimeMs","size","Buffer","byteLength","text","presets","doc","extracted","identified","map","j","report","i","batchCap","batch","slice","vectors","embedDocuments","writes","job","q","Int8Array","round","vector","from","buffer","transaction","writeVectors","currentStmt","join","current","flatMap","identities","Number","some","tick","finish","ensureDocumentEmbeddings","_cfg","prepareSemanticQuery","terms","rootDir","baseDir","match","t","includes","queryVector","embedQuery","qv","semanticCandidates","prepared","fetch","candidates","hasEmbedding","hasVector","similarNotes","opts","similar"],"mappings":"AAAA,SAAsBA,gBAAgB,QAA6B,qBAAqB;AACxF,SAASC,UAAU,QAAQ,eAAe;AAE1C,SAASC,KAAK,QAAQ,uBAAuB;AAC7C,SAASC,QAAQ,QAAQ,uBAAuB;AAChD,SAASC,QAAQ,QAAQ,wBAAwB;AACjD,SAASC,WAAW,QAAQ,mBAAmB;AAC/C,SAASC,0BAA0B,QAAQ,wBAAwB;AACnE,SAASC,OAAO,QAAQ,qBAAqB;AAC7C,SAASC,4BAA4B,QAAQ,wBAAwB;AAErE,SAASC,iBAAiB,EAAEC,aAAa,QAAQ,eAAe;AAChE,SAASC,gBAAgB,QAAQ,eAAe;AAChD,SAASC,WAAW,QAAQ,gBAAgB;AAC5C,SAAgCC,UAAU,QAAQ,aAAa;AAE/D,yFAAyF;AACzF,qEAAqE;AACrE,OAAO,SAASC,QAAQC,IAAkB,EAAEC,IAAY,EAAEC,IAAa;IACrE,MAAMC,IAAI,IAAIC,aAAaH;IAC3B,IAAII,OAAO;IACX,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKD,QAAQL,IAAI,CAACM,EAAE,GAAGN,IAAI,CAACM,EAAE;IACxDD,OAAOE,KAAKC,IAAI,CAACH,QAAQ;IACzB,IAAK,IAAIC,IAAI,GAAGA,IAAIL,MAAMK,IAAKH,CAAC,CAACG,EAAE,GAAGN,IAAI,CAACM,EAAE,GAAGD;IAChD,IAAI,CAACH,MAAM,OAAO;QAAEC;QAAGM,OAAO;IAAE;IAChC,IAAIC,MAAM;IACV,IAAK,IAAIJ,IAAI,GAAGA,IAAIL,MAAMK,IAAKI,MAAMH,KAAKG,GAAG,CAACA,KAAKH,KAAKI,GAAG,CAACR,CAAC,CAACG,EAAE;IAChE,OAAO;QAAEH;QAAGM,OAAOC,MAAM,OAAO;IAAE;AACpC;AAEA,gGAAgG;AAChG,gGAAgG;AAChG,wDAAwD;AACxD,SAASE,mBAAmBC,MAAc;IACxC,OAAO,IAAI3B,WAAW,mBAAmB,GAAG2B,OAAO,8FAA8F,CAAC;AACpJ;AAEA,gGAAgG;AAChG,0FAA0F;AAC1F,OAAO,eAAeC,aAAaC,KAAY,EAAEC,GAAW,EAAEC,OAA6B,EAAEC,iBAAiC,EAAEC,UAA4B,CAAC,CAAC;QA4BzIxB;QA3BnBwB,iBAEAA,kBAOAA,kBAIAA,kBAuCA,sFAAsF;IACtF,2EAA2E;IAC3EA,kBAMAA;KA5DAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMC,2BAA2BJ,sBAAsBK,YAAY,MAAM/B,QAAQuB,OAAOxB,8BAA8B2B;KACtHC,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAI,CAAC5B,6BAA6B6B,0BAA0BrC,iBAAiB+B,KAAK5B,YAAY;QAC5F,MAAMwB,mBAAmB;IAC3B;IAEA,MAAMY,cAAc,MAAMT,MAAMU,OAAO,CAAC;IACxC,MAAMC,QAAQ,AAAE,CAAA,MAAMF,YAAYG,GAAG,EAAC,EAAwEC,MAAM,CAAC,CAACC;;uBAAQZ,oBAAAA,8BAAAA,QAASa,GAAG,CAACD,IAAIE,IAAI,wCAAK;;KACxJZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,IAAIK,MAAMM,MAAM,KAAK,GAAG;IAExB,MAAMC,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMa,qBAAqBjD,iBAAiB+B,KAAK5B;IACjD,IAAI,CAACK,6BAA6B6B,0BAA0BY,qBAAqB;QAC/E,MAAMtB,mBAAmB;IAC3B;IACA,MAAMuB,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IAEpD,MAAMoC,SAAS,IAAIC;IACnB,KAAK,MAAMT,OAAOH,MAAO;YACVW;QAAb,MAAME,QAAOF,cAAAA,OAAOG,GAAG,CAACX,IAAIE,IAAI,eAAnBM,yBAAAA,cAAwB,EAAE;QACvCE,KAAKE,IAAI,CAACZ;QACVQ,OAAOK,GAAG,CAACb,IAAIE,IAAI,EAAEQ;IACvB;IAEA,MAAMI,cAAahD,iBAAAA,cAAcoB,oBAAdpB,4BAAAA,iBAAwB,IAAI2C;IAC/C,MAAMM,aAAa,MAAM7B,MAAMU,OAAO,CAAC;IAEvC,MAAMoB,OAA+E,EAAE;IACvF,KAAK,MAAM,CAACd,MAAMe,QAAQ,IAAIT,OAAQ;YASpClB;QARA,MAAM4B,QAAQJ,WAAWH,GAAG,CAACT;QAC7B,IAAIgB,SAASD,QAAQE,KAAK,CAAC,CAACnB;gBAAQkB;mBAAAA,EAAAA,mBAAAA,KAAK,CAAClB,IAAIoB,KAAK,CAAC,cAAhBF,uCAAAA,iBAAkBG,QAAQ,MAAKrB,IAAIsB,gBAAgB;YAAG;YACxF,KAAK,MAAMtB,OAAOiB,QAASD,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGF,KAAK,CAAClB,IAAIoB,KAAK,CAAC;YAAC;YACnF;QACF;QACA,2FAA2F;QAC3F,8EAA8E;QAC9E,MAAMG,SAAU,MAAMR,WAAWJ,GAAG,CAACT;SACrCZ,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAI,CAAC+B,QAAQ,MAAM,IAAIlE,WAAW,mBAAmB,CAAC,mCAAmC,EAAE6C,KAAK,kGAAkG,CAAC;QACnM,MAAMsB,SAAS/D,YAAY;YAAEgE,SAASvB;YAAMwB,SAASxB;YAAMyB,SAAS;YAAGC,SAAS;YAAGC,MAAMC,OAAOC,UAAU,CAACR,OAAOS,IAAI;YAAGC,SAAS,EAAE;YAAE3E,OAAO;QAAK,GAAGiE,OAAOS,IAAI,EAAE;YAAC1E;SAAM,EAAE6B,KAAK+C,GAAG,CAACC,SAAS,CAAC7E,KAAK;QACnM,KAAK,MAAM0C,OAAOiB,QAAS;YACzB,MAAMG,QAAQI,MAAM,CAACxB,IAAIoB,KAAK,CAAC;YAC/B,IAAI,CAACA,OAAO,MAAM,IAAI/D,WAAW,mBAAmB,CAAC,4BAA4B,EAAE6C,KAAK,2GAA2G,CAAC;YACpM,MAAMkC,aAAavE,kBAAkBuD,MAAMY,IAAI;YAC/C,IAAII,WAAWf,QAAQ,KAAKrB,IAAIsB,gBAAgB,EAAE,MAAMvC,mBAAmB,CAAC,cAAc,EAAEmB,KAAK,CAAC,EAAEF,IAAIoB,KAAK,CAAC,yCAAyC,CAAC;YACxJJ,KAAKJ,IAAI,CAAC;gBAAEV;gBAAMkB,OAAOpB,IAAIoB,KAAK;gBAAE,GAAGgB,UAAU;YAAC;QACpD;IACF;KAIA9C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMzB,iBACJmB,OACAkB,UACAY,KAAKqB,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI;KAExB1C,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAE9B,0FAA0F;IAC1F,4BAA4B;IAC5B,MAAM+C,SAAS/E,SAAS,oBAAoBwD,KAAKb,MAAM;IACvD,IAAK,IAAIqC,IAAI,GAAGA,IAAIxB,KAAKb,MAAM,EAAEqC,KAAKpC,SAASqC,QAAQ,CAAE;YAEvDnD,kBAMAA,kBAQAA;QAfA,MAAMoD,QAAQ1B,KAAK2B,KAAK,CAACH,GAAGA,IAAIpC,SAASqC,QAAQ;SACjDnD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM6D,UAAU,MAAMxC,SAASyC,cAAc,CAC3CH,MAAML,GAAG,CAAC,CAACC,IAAMA,EAAEN,IAAI,GACvB1C;SAEFA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,IAAIpC,iBAAiB+B,KAAK5B,cAAc8C,oBAAoB,MAAMtB,mBAAmB;QACrF,MAAM+D,SAASJ,MAAML,GAAG,CAAC,CAACU,KAAKT;YAC7B,MAAM,EAAEhE,CAAC,EAAEM,KAAK,EAAE,GAAGV,QAAQ0E,OAAO,CAACN,EAAE,EAAEhC,WAAW;YACpD,MAAM0C,IAAI,IAAIC,UAAU3C;YACxB,IAAK,IAAI7B,IAAI,GAAGA,IAAI6B,WAAW7B,IAAKuE,CAAC,CAACvE,EAAE,GAAGC,KAAKwE,KAAK,CAAC5E,CAAC,CAACG,EAAE,GAAGG;YAC7D,OAAO;gBAAEsB,MAAM6C,IAAI7C,IAAI;gBAAEkB,OAAO2B,IAAI3B,KAAK;gBAAExC;gBAAOuE,QAAQrB,OAAOsB,IAAI,CAACJ,EAAEK,MAAM;YAAE;QAClF;SACA/D,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAC9B,MAAMN,MAAMoE,WAAW,CAAC;gBACtB,wFAAwF;YACxF,mFAAmF;YACnFhE,iBAEAA,kBAMAA;aARAA,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;YAC9B,MAAMN,MAAM0D,OAAO,CAACW,YAAY,CAACT;aACjCxD,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;YAC9B,IAAI,AAAC,MAAM7B,QAAQuB,OAAOxB,gCAAiC+B,0BAA0B,MAAMV,mBAAmB;YAC9G,MAAMyE,cAAc,MAAMtE,MAAMU,OAAO,CAAC,CAAC,6DAA6D,EAAE8C,MAAML,GAAG,CAAC,IAAM,8BAA8BoB,IAAI,CAAC,SAAS;YACpK,MAAMC,UAAW,MAAMF,YAAY1D,GAAG,IAAI4C,MAAMiB,OAAO,CAAC,CAACZ,MAAQ;oBAACA,IAAI7C,IAAI;oBAAE6C,IAAI3B,KAAK;iBAAC;YACtF,MAAMwC,aAAa,IAAInD,IAAIiD,QAAQrB,GAAG,CAAC,CAACrC,MAAQ;oBAAC,GAAGA,IAAIE,IAAI,CAAC,EAAE,EAAE2D,OAAO7D,IAAIoB,KAAK,GAAG;oBAAEpB,IAAIsB,gBAAgB;iBAAC;YAC3G,IAAIoB,MAAMoB,IAAI,CAAC,CAACf,MAAQa,WAAWjD,GAAG,CAAC,GAAGoC,IAAI7C,IAAI,CAAC,EAAE,EAAE6C,IAAI3B,KAAK,EAAE,MAAM2B,IAAI1B,QAAQ,GAAG,MAAMtC,mBAAmB;aAChHO,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;QAChC;QACA+C,OAAOwB,IAAI,CAACrF,KAAK6B,GAAG,CAACiC,IAAIpC,SAASqC,QAAQ,EAAEzB,KAAKb,MAAM;IACzD;IACAoC,OAAOyB,MAAM;AACf;AAEA,OAAO,eAAeC,yBAAyB/E,KAAY,EAAEgF,IAAY,EAAE9E,OAA4B;IACrG,MAAM6B,UAAU,AAAC,CAAA,MAAM/B,MAAM0D,OAAO,CAAC3B,OAAO,EAAC,EAAGlB,MAAM,CAAC,CAACC,MAAQZ,QAAQa,GAAG,CAACD,IAAIE,IAAI;IACpF,IAAIe,QAAQd,MAAM,GAAG,GAAG;QACtB,MAAM,IAAI9C,WAAW,mBAAmB,GAAG4D,QAAQd,MAAM,CAAC,+IAA+I,CAAC;IAC5M;AACF;AAOA,2FAA2F;AAC3F,2FAA2F;AAC3F,OAAO,eAAegE,qBAAqBhF,GAAW,EAAEiF,KAAa,EAAE9E,UAA4B,CAAC,CAAC;QACnF,cAEF8E;QACd9E,iBAEAA,kBAGAA;IARA,MAAM+E,WAAU,eAAA,AAAClF,IAAgCkF,OAAO,cAAxC,0BAAA,eAA4C,AAAClF,IAAgCmF,OAAO;IACpG,IAAI,CAACD,SAAS,MAAM,IAAIhH,WAAW,eAAe;IAClD,MAAM2E,OAAO,EAACoC,eAAAA,MAAMG,KAAK,CAAC,gCAAZH,0BAAAA,eAAkC,EAAE,EAAErE,MAAM,CAAC,CAACyE,IAAM,CAAC;YAAC;YAAO;YAAM;YAAO;SAAO,CAACC,QAAQ,CAACD,IAAIf,IAAI,CAAC;KAClHnE,kBAAAA,QAAQC,MAAM,cAAdD,sCAAAA,gBAAgBE,cAAc;IAC9B,MAAMY,WAAW,MAAMpC,YAAYmB,KAAKG;KACxCA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAMc,YAAY5B,KAAK6B,GAAG,CAACtC,YAAYmC,SAAShC,IAAI;IACpD,MAAMsG,cAAc,MAAMtE,SAASuE,UAAU,CAAC3C,MAAM1C;KACpDA,mBAAAA,QAAQC,MAAM,cAAdD,uCAAAA,iBAAgBE,cAAc;IAC9B,MAAM,EAAElB,GAAGsG,EAAE,EAAE,GAAG1G,QAAQwG,aAAapE,WAAW;IAClD,OAAO;QAAE6C,QAAQyB;QAAItE;IAAU;AACjC;AAEA,gGAAgG;AAChG,8FAA8F;AAC9F,OAAO,SAASuE,mBAAmB3F,KAAY,EAAE4F,QAA+B,EAAEC,KAAa,EAAE3F,OAAqB;IACpH,OAAOF,MAAM0D,OAAO,CAACoC,UAAU,CAACF,SAAS3B,MAAM,EAAE2B,SAASxE,SAAS,EAAEyE,OAAO3F;AAC9E;AAEA,6FAA6F;AAC7F,mEAAmE;AACnE,OAAO,eAAe6F,aAAa/F,KAAY,EAAEgB,IAAY;IAC3D,OAAOhB,MAAM0D,OAAO,CAACsC,SAAS,CAAChF;AACjC;AAEA,+FAA+F;AAC/F,0BAA0B;AAC1B,OAAO,eAAeiF,aAAajG,KAAY,EAAEgF,IAAY,EAAEhE,IAAY,EAAEkF,IAAgE;IAC3I,OAAOlG,MAAM0D,OAAO,CAACyC,OAAO,CAACnF,MAAMkF;AACrC"}

package/dist/esm/embed/registry.d.ts

@@ -1,3 +1,3 @@
 import type { Config } from '../config/index.js';
-import type { EmbedProvider } from './types.js';
-export declare function getProvider(cfg: Config): Promise<EmbedProvider>;
+import type { EmbedCallOptions, EmbedProvider } from './types.js';
+export declare function getProvider(cfg: Config, options?: EmbedCallOptions): Promise<EmbedProvider>;

package/dist/esm/embed/registry.js

@@ -1,6 +1,7 @@
 import { embedConfig } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { cohereProvider } from './cohere.js';
+import { effectiveEmbedEndpoint } from './endpoint.js';
 import { openaiProvider } from './openai.js';
 import { staticProvider } from './static.js';
 // One factory per wire protocol: adding a protocol is a new file plus an entry here, not
@@ -11,24 +12,52 @@ const REGISTRY = {
     cohere: (model, url, key)=>cohereProvider(model, url, key)
 };
 const providers = new Map();
-export function getProvider(cfg) {
-    var _e_url, _e_key, _e_languages;
+function aborted(reason) {
+    return Promise.reject(reason);
+}
+function waitForProvider(provider, signal) {
+    if (signal.aborted) return aborted(signal.reason);
+    return new Promise((resolve, reject)=>{
+        const cleanup = ()=>signal.removeEventListener('abort', onAbort);
+        const onAbort = ()=>{
+            cleanup();
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', onAbort, {
+            once: true
+        });
+        provider.then((value)=>{
+            cleanup();
+            resolve(value);
+        }, (err)=>{
+            cleanup();
+            reject(err);
+        });
+        if (signal.aborted) onAbort();
+    });
+}
+export function getProvider(cfg, options = {}) {
+    var _e_key, _e_languages;
+    var _options_signal;
+    // A caller that has already left must not start shared model initialization.
+    if ((_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.aborted) return aborted(options.signal.reason);
     const e = embedConfig(cfg);
     if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an "embed" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');
     const factory = REGISTRY[e.provider];
     if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider "${e.provider}" has no provider implementation in this build`);
+    const endpoint = effectiveEmbedEndpoint(e.provider, e.url);
     // Every input that shapes the provider is in the key: two trees sharing an endpoint and
     // model but declaring different languages or key env vars are two providers, not one.
     const sig = JSON.stringify([
         e.provider,
         e.model,
-        (_e_url = e.url) !== null && _e_url !== void 0 ? _e_url : '',
+        endpoint !== null && endpoint !== void 0 ? endpoint : '',
         (_e_key = e.key) !== null && _e_key !== void 0 ? _e_key : '',
         (_e_languages = e.languages) !== null && _e_languages !== void 0 ? _e_languages : []
     ]);
     let p = providers.get(sig);
     if (!p) {
-        p = factory(e.model, e.url, e.key);
+        p = factory(e.model, endpoint, e.key);
         // Owner-declared languages override a provider's own: the config is the one home for
         // what no model card can state (http models have no fetchable card here).
         if (e.languages) p = p.then((prov)=>({
@@ -40,5 +69,5 @@ export function getProvider(cfg) {
         // poisoning this config for the process lifetime.
         p.catch(()=>providers.delete(sig));
     }
-    return p;
+    return options.signal ? waitForProvider(p, options.signal) : p;
 }

package/dist/esm/embed/registry.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nexport function getProvider(cfg: Config): Promise<EmbedProvider> {\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, e.url ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, e.url, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","getProvider","cfg","e","factory","provider","sig","JSON","stringify","languages","p","get","then","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQR,eAAeK,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,OAAO,SAASC,YAAYC,GAAW;QAOYC,QAAaA,QAAaA;IAN3E,MAAMA,IAAIhB,YAAYe;IACtB,IAAI,CAACC,GAAG,MAAM,IAAIf,WAAW,kBAAkB;IAC/C,MAAMgB,UAAUZ,QAAQ,CAACW,EAAEE,QAAQ,CAAC;IACpC,IAAI,CAACD,SAAS,MAAM,IAAIhB,WAAW,eAAe,CAAC,gBAAgB,EAAEe,EAAEE,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,wFAAwF;IACxF,sFAAsF;IACtF,MAAMC,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEE,QAAQ;QAAEF,EAAET,KAAK;SAAES,SAAAA,EAAEP,GAAG,cAALO,oBAAAA,SAAS;SAAIA,SAAAA,EAAEN,GAAG,cAALM,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAC7F,IAAIO,IAAIX,UAAUY,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAET,KAAK,EAAES,EAAEP,GAAG,EAAEO,EAAEN,GAAG;QACjC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIM,EAAEM,SAAS,EAAEC,IAAIA,EAAEE,IAAI,CAAC,CAACC,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEJ,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzEV,UAAUe,GAAG,CAACR,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEK,KAAK,CAAC,IAAMhB,UAAUiB,MAAM,CAACV;IACjC;IACA,OAAOI;AACT"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/registry.ts"],"sourcesContent":["import type { Config } from '../config/index.ts';\nimport { embedConfig } from '../config/index.ts';\nimport { SenseError } from '../errors.ts';\nimport { cohereProvider } from './cohere.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { openaiProvider } from './openai.ts';\nimport { staticProvider } from './static.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\ntype ProviderFactory = (model: string, url: string | undefined, key: string | undefined) => Promise<EmbedProvider>;\n\n// One factory per wire protocol: adding a protocol is a new file plus an entry here, not\n// an edit to a dispatch ternary.\nconst REGISTRY: Record<string, ProviderFactory> = {\n  static: (model) => staticProvider(model),\n  openai: (model, url, key) => openaiProvider(model, url, key),\n  cohere: (model, url, key) => cohereProvider(model, url, key),\n};\n\nconst providers = new Map<string, Promise<EmbedProvider>>();\n\nfunction aborted(reason: unknown): Promise<never> {\n  return Promise.reject(reason);\n}\n\nfunction waitForProvider(provider: Promise<EmbedProvider>, signal: AbortSignal): Promise<EmbedProvider> {\n  if (signal.aborted) return aborted(signal.reason);\n  return new Promise((resolve, reject) => {\n    const cleanup = () => signal.removeEventListener('abort', onAbort);\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    provider.then(\n      (value) => {\n        cleanup();\n        resolve(value);\n      },\n      (err: unknown) => {\n        cleanup();\n        reject(err);\n      }\n    );\n    if (signal.aborted) onAbort();\n  });\n}\n\nexport function getProvider(cfg: Config, options: EmbedCallOptions = {}): Promise<EmbedProvider> {\n  // A caller that has already left must not start shared model initialization.\n  if (options.signal?.aborted) return aborted(options.signal.reason);\n  const e = embedConfig(cfg);\n  if (!e) throw new SenseError('EMBED_DISABLED', 'this tree has no embedding model: add an \"embed\" block naming one to sense.config.json; a Hugging Face id fetches automatically at first use, or run `sense download` to prefetch it');\n  const factory = REGISTRY[e.provider];\n  if (!factory) throw new SenseError('EMBED_MODEL', `embed.provider \"${e.provider}\" has no provider implementation in this build`);\n  const endpoint = effectiveEmbedEndpoint(e.provider, e.url);\n  // Every input that shapes the provider is in the key: two trees sharing an endpoint and\n  // model but declaring different languages or key env vars are two providers, not one.\n  const sig = JSON.stringify([e.provider, e.model, endpoint ?? '', e.key ?? '', e.languages ?? []]);\n  let p = providers.get(sig);\n  if (!p) {\n    p = factory(e.model, endpoint, e.key);\n    // Owner-declared languages override a provider's own: the config is the one home for\n    // what no model card can state (http models have no fetchable card here).\n    if (e.languages) p = p.then((prov) => ({ ...prov, languages: e.languages }));\n    providers.set(sig, p);\n    // A rejected construction is evicted, so a transient failure retries instead of\n    // poisoning this config for the process lifetime.\n    p.catch(() => providers.delete(sig));\n  }\n  return options.signal ? waitForProvider(p, options.signal) : p;\n}\n"],"names":["embedConfig","SenseError","cohereProvider","effectiveEmbedEndpoint","openaiProvider","staticProvider","REGISTRY","static","model","openai","url","key","cohere","providers","Map","aborted","reason","Promise","reject","waitForProvider","provider","signal","resolve","cleanup","removeEventListener","onAbort","addEventListener","once","then","value","err","getProvider","cfg","options","e","factory","endpoint","sig","JSON","stringify","languages","p","get","prov","set","catch","delete"],"mappings":"AACA,SAASA,WAAW,QAAQ,qBAAqB;AACjD,SAASC,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,QAAQ,cAAc;AAC7C,SAASC,cAAc,QAAQ,cAAc;AAK7C,yFAAyF;AACzF,iCAAiC;AACjC,MAAMC,WAA4C;IAChDC,QAAQ,CAACC,QAAUH,eAAeG;IAClCC,QAAQ,CAACD,OAAOE,KAAKC,MAAQP,eAAeI,OAAOE,KAAKC;IACxDC,QAAQ,CAACJ,OAAOE,KAAKC,MAAQT,eAAeM,OAAOE,KAAKC;AAC1D;AAEA,MAAME,YAAY,IAAIC;AAEtB,SAASC,QAAQC,MAAe;IAC9B,OAAOC,QAAQC,MAAM,CAACF;AACxB;AAEA,SAASG,gBAAgBC,QAAgC,EAAEC,MAAmB;IAC5E,IAAIA,OAAON,OAAO,EAAE,OAAOA,QAAQM,OAAOL,MAAM;IAChD,OAAO,IAAIC,QAAQ,CAACK,SAASJ;QAC3B,MAAMK,UAAU,IAAMF,OAAOG,mBAAmB,CAAC,SAASC;QAC1D,MAAMA,UAAU;YACdF;YACAL,OAAOG,OAAOL,MAAM;QACtB;QACAK,OAAOK,gBAAgB,CAAC,SAASD,SAAS;YAAEE,MAAM;QAAK;QACvDP,SAASQ,IAAI,CACX,CAACC;YACCN;YACAD,QAAQO;QACV,GACA,CAACC;YACCP;YACAL,OAAOY;QACT;QAEF,IAAIT,OAAON,OAAO,EAAEU;IACtB;AACF;AAEA,OAAO,SAASM,YAAYC,GAAW,EAAEC,UAA4B,CAAC,CAAC;QAUJC,QAAaA;QAR1ED;IADJ,6EAA6E;IAC7E,KAAIA,kBAAAA,QAAQZ,MAAM,cAAdY,sCAAAA,gBAAgBlB,OAAO,EAAE,OAAOA,QAAQkB,QAAQZ,MAAM,CAACL,MAAM;IACjE,MAAMkB,IAAIlC,YAAYgC;IACtB,IAAI,CAACE,GAAG,MAAM,IAAIjC,WAAW,kBAAkB;IAC/C,MAAMkC,UAAU7B,QAAQ,CAAC4B,EAAEd,QAAQ,CAAC;IACpC,IAAI,CAACe,SAAS,MAAM,IAAIlC,WAAW,eAAe,CAAC,gBAAgB,EAAEiC,EAAEd,QAAQ,CAAC,8CAA8C,CAAC;IAC/H,MAAMgB,WAAWjC,uBAAuB+B,EAAEd,QAAQ,EAAEc,EAAExB,GAAG;IACzD,wFAAwF;IACxF,sFAAsF;IACtF,MAAM2B,MAAMC,KAAKC,SAAS,CAAC;QAACL,EAAEd,QAAQ;QAAEc,EAAE1B,KAAK;QAAE4B,qBAAAA,sBAAAA,WAAY;SAAIF,SAAAA,EAAEvB,GAAG,cAALuB,oBAAAA,SAAS;SAAIA,eAAAA,EAAEM,SAAS,cAAXN,0BAAAA,eAAe,EAAE;KAAC;IAChG,IAAIO,IAAI5B,UAAU6B,GAAG,CAACL;IACtB,IAAI,CAACI,GAAG;QACNA,IAAIN,QAAQD,EAAE1B,KAAK,EAAE4B,UAAUF,EAAEvB,GAAG;QACpC,qFAAqF;QACrF,0EAA0E;QAC1E,IAAIuB,EAAEM,SAAS,EAAEC,IAAIA,EAAEb,IAAI,CAAC,CAACe,OAAU,CAAA;gBAAE,GAAGA,IAAI;gBAAEH,WAAWN,EAAEM,SAAS;YAAC,CAAA;QACzE3B,UAAU+B,GAAG,CAACP,KAAKI;QACnB,gFAAgF;QAChF,kDAAkD;QAClDA,EAAEI,KAAK,CAAC,IAAMhC,UAAUiC,MAAM,CAACT;IACjC;IACA,OAAOJ,QAAQZ,MAAM,GAAGF,gBAAgBsB,GAAGR,QAAQZ,MAAM,IAAIoB;AAC/D"}

package/dist/esm/embed/static.js

@@ -68,7 +68,15 @@ export async function staticProvider(model, root) {
         dims,
         batchCap: BATCH_CAP,
         languages,
-        embedDocuments: async (texts)=>texts.map(one),
-        embedQuery: async (text)=>one(text)
+        embedDocuments: async (texts, options)=>{
+            var _options_signal;
+            options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            return texts.map(one);
+        },
+        embedQuery: async (text, options)=>{
+            var _options_signal;
+            options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            return one(text);
+        }
     };
 }