sensemaking 0.25.1 → 0.26.0

package/dist/esm/embed/http.js

@@ -1,33 +1,153 @@
+import { SenseError } from '../errors.js';
 const MAX_ATTEMPTS = 3;
-function sleep(ms) {
-    return new Promise((resolve)=>setTimeout(resolve, ms));
+const MAX_RETRY_AFTER_MS = 10000;
+const DEFAULT_BASE_DELAY_MS = 200;
+// Initial policy bounds small API/metadata responses separately from model-file downloads.
+export const HTTP_ATTEMPT_TIMEOUT_MS = 30000;
+export const MODEL_FILE_ATTEMPT_TIMEOUT_MS = 5 * 60000;
+function safeUrl(url) {
+    try {
+        const parsed = new URL(url);
+        parsed.username = '';
+        parsed.password = '';
+        parsed.search = '';
+        parsed.hash = '';
+        return parsed.href;
+    } catch  {
+        return 'the configured embedding endpoint';
+    }
+}
+function attemptController(callerSignal, timeoutMs) {
+    callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.throwIfAborted();
+    const controller = new AbortController();
+    const timeoutReason = {};
+    const relayAbort = ()=>controller.abort(callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.reason);
+    callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.addEventListener('abort', relayAbort, {
+        once: true
+    });
+    if (callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.aborted) relayAbort();
+    const timer = setTimeout(()=>controller.abort(timeoutReason), timeoutMs);
+    return {
+        controller,
+        timeoutReason,
+        cleanup: ()=>{
+            clearTimeout(timer);
+            callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.removeEventListener('abort', relayAbort);
+        }
+    };
+}
+function disposeResponse(response, controller) {
+    const cancellation = response.body && !response.bodyUsed ? response.body.cancel() : undefined;
+    controller.abort();
+    // The controller owns socket cleanup; observe stream cancellation without delaying the attempt.
+    if (cancellation) void cancellation.then(()=>undefined, ()=>undefined);
+}
+function sleep(ms, signal) {
+    signal === null || signal === void 0 ? void 0 : signal.throwIfAborted();
+    if (!signal) return new Promise((resolve)=>setTimeout(resolve, ms));
+    return new Promise((resolve, reject)=>{
+        let timer;
+        const cleanup = ()=>{
+            if (timer) clearTimeout(timer);
+            signal.removeEventListener('abort', onAbort);
+        };
+        const onAbort = ()=>{
+            cleanup();
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', onAbort, {
+            once: true
+        });
+        timer = setTimeout(()=>{
+            cleanup();
+            resolve();
+        }, ms);
+        if (signal.aborted) onAbort();
+    });
+}
+function failedRequest(url, phase, attempts, timeoutMs, timedOut) {
+    const endpoint = safeUrl(url);
+    if (timedOut) {
+        return new SenseError('EMBED_MODEL', `${endpoint} timed out after ${timeoutMs}ms while waiting for ${phase} (attempt ${attempts}/${MAX_ATTEMPTS}); check the embedding service and network`);
+    }
+    return new SenseError('EMBED_MODEL', `${endpoint} failed while waiting for ${phase} after ${attempts} attempts; check the embedding service and network`);
+}
+function buffered(response, body, requestUrl) {
+    return {
+        ok: response.ok,
+        status: response.status,
+        headers: response.headers,
+        body,
+        url: safeUrl(response.url || requestUrl)
+    };
 }
-// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential
-// backoff honoring Retry-After; any other status returns immediately for the caller to handle.
-export async function fetchWithRetry(url, init, baseDelayMs = 200) {
+// Each attempt covers response headers and the complete successful body. Retry cleanup and
+// backoff honor one caller signal without transferring ownership of that signal.
+export async function fetchWithRetry(url, init, options = {}) {
+    var _options_attemptTimeoutMs, _options_baseDelayMs;
+    const timeoutMs = (_options_attemptTimeoutMs = options.attemptTimeoutMs) !== null && _options_attemptTimeoutMs !== void 0 ? _options_attemptTimeoutMs : HTTP_ATTEMPT_TIMEOUT_MS;
+    const baseDelayMs = (_options_baseDelayMs = options.baseDelayMs) !== null && _options_baseDelayMs !== void 0 ? _options_baseDelayMs : DEFAULT_BASE_DELAY_MS;
     for(let attempt = 0; attempt < MAX_ATTEMPTS; attempt++){
+        var _options_signal;
+        (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+        const { controller, timeoutReason, cleanup } = attemptController(options.signal, timeoutMs);
+        let response;
+        let phase = 'response headers';
+        let retryDelayMs;
         try {
-            const res = await fetch(url, init);
-            if (res.ok || res.status !== 429 && res.status < 500 || attempt === MAX_ATTEMPTS - 1) return res;
-            const retryAfter = Number(res.headers.get('retry-after'));
-            // Capped: a server naming a huge Retry-After must not stall an embed run for it.
-            await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10000) : baseDelayMs * 2 ** attempt);
-        } catch (err) {
-            if (attempt === MAX_ATTEMPTS - 1) throw err;
-            await sleep(baseDelayMs * 2 ** attempt);
+            response = await fetch(url, {
+                ...init,
+                signal: controller.signal
+            });
+            if (!response.ok) {
+                const retryable = response.status === 429 || response.status >= 500;
+                if (retryable && attempt < MAX_ATTEMPTS - 1) {
+                    const retryAfter = Number(response.headers.get('retry-after'));
+                    retryDelayMs = retryAfter > 0 ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) : baseDelayMs * 2 ** attempt;
+                    disposeResponse(response, controller);
+                } else {
+                    var _options_signal1;
+                    disposeResponse(response, controller);
+                    (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+                    return buffered(response, new ArrayBuffer(0), url);
+                }
+            } else {
+                var _options_signal2;
+                phase = 'response body';
+                const body = await response.arrayBuffer();
+                (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
+                return buffered(response, body, url);
+            }
+        } catch  {
+            try {
+                var _options_signal3;
+                (_options_signal3 = options.signal) === null || _options_signal3 === void 0 ? void 0 : _options_signal3.throwIfAborted();
+            } finally{
+                if (response) disposeResponse(response, controller);
+            }
+            const timedOut = controller.signal.reason === timeoutReason;
+            if (attempt === MAX_ATTEMPTS - 1) throw failedRequest(url, phase, attempt + 1, timeoutMs, timedOut);
+            retryDelayMs = baseDelayMs * 2 ** attempt;
+        } finally{
+            cleanup();
         }
+        await sleep(retryDelayMs !== null && retryDelayMs !== void 0 ? retryDelayMs : baseDelayMs * 2 ** attempt, options.signal);
     }
     throw new Error('unreachable');
 }
-// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as
-// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.
+// Any HTTP response means the endpoint is up. Abort it immediately because status does not
+// consume its body, and bound the headers wait for an air-gapped machine.
 export async function probeReachable(url, timeoutMs = 1500) {
+    const { controller, cleanup } = attemptController(undefined, timeoutMs);
     try {
-        await fetch(url, {
-            signal: AbortSignal.timeout(timeoutMs)
+        const response = await fetch(url, {
+            signal: controller.signal
         });
+        disposeResponse(response, controller);
         return true;
     } catch  {
         return false;
+    } finally{
+        cleanup();
     }
 }

package/dist/esm/embed/http.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    try {\n      const res = await fetch(url, init);\n      if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n      const retryAfter = Number(res.headers.get('retry-after'));\n      // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n      await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n    } catch (err) {\n      if (attempt === MAX_ATTEMPTS - 1) throw err;\n      await sleep(baseDelayMs * 2 ** attempt);\n    }\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n"],"names":["MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","fetchWithRetry","url","init","baseDelayMs","attempt","res","fetch","ok","status","retryAfter","Number","headers","get","Math","min","err","Error","probeReachable","timeoutMs","signal","AbortSignal","timeout"],"mappings":"AAAA,MAAMA,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,CAACC,UAAYC,WAAWD,SAASF;AACtD;AAEA,2FAA2F;AAC3F,+FAA+F;AAC/F,OAAO,eAAeI,eAAeC,GAAW,EAAEC,IAAiB,EAAEC,cAAc,GAAG;IACpF,IAAK,IAAIC,UAAU,GAAGA,UAAUV,cAAcU,UAAW;QACvD,IAAI;YACF,MAAMC,MAAM,MAAMC,MAAML,KAAKC;YAC7B,IAAIG,IAAIE,EAAE,IAAKF,IAAIG,MAAM,KAAK,OAAOH,IAAIG,MAAM,GAAG,OAAQJ,YAAYV,eAAe,GAAG,OAAOW;YAC/F,MAAMI,aAAaC,OAAOL,IAAIM,OAAO,CAACC,GAAG,CAAC;YAC1C,iFAAiF;YACjF,MAAMjB,MAAMc,aAAa,IAAII,KAAKC,GAAG,CAACL,aAAa,MAAM,SAAUN,cAAc,KAAKC;QACxF,EAAE,OAAOW,KAAK;YACZ,IAAIX,YAAYV,eAAe,GAAG,MAAMqB;YACxC,MAAMpB,MAAMQ,cAAc,KAAKC;QACjC;IACF;IACA,MAAM,IAAIY,MAAM;AAClB;AAEA,8FAA8F;AAC9F,2FAA2F;AAC3F,OAAO,eAAeC,eAAehB,GAAW,EAAEiB,YAAY,IAAI;IAChE,IAAI;QACF,MAAMZ,MAAML,KAAK;YAAEkB,QAAQC,YAAYC,OAAO,CAACH;QAAW;QAC1D,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT;AACF"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\nconst MAX_ATTEMPTS = 3;\nconst MAX_RETRY_AFTER_MS = 10_000;\nconst DEFAULT_BASE_DELAY_MS = 200;\n\n// Initial policy bounds small API/metadata responses separately from model-file downloads.\nexport const HTTP_ATTEMPT_TIMEOUT_MS = 30_000;\nexport const MODEL_FILE_ATTEMPT_TIMEOUT_MS = 5 * 60_000;\n\nexport type FetchRequestInit = Omit<RequestInit, 'signal'> & { signal?: never };\n\nexport interface FetchRetryOptions {\n  signal?: AbortSignal;\n  attemptTimeoutMs?: number;\n  baseDelayMs?: number;\n}\n\nexport interface BufferedResponse {\n  readonly ok: boolean;\n  readonly status: number;\n  readonly headers: Headers;\n  readonly body: ArrayBuffer;\n  readonly url: string;\n}\n\ntype ResponsePhase = 'response headers' | 'response body';\n\nfunction safeUrl(url: string): string {\n  try {\n    const parsed = new URL(url);\n    parsed.username = '';\n    parsed.password = '';\n    parsed.search = '';\n    parsed.hash = '';\n    return parsed.href;\n  } catch {\n    return 'the configured embedding endpoint';\n  }\n}\n\nfunction attemptController(\n  callerSignal: AbortSignal | undefined,\n  timeoutMs: number\n): {\n  controller: AbortController;\n  timeoutReason: object;\n  cleanup: () => void;\n} {\n  callerSignal?.throwIfAborted();\n  const controller = new AbortController();\n  const timeoutReason = {};\n  const relayAbort = () => controller.abort(callerSignal?.reason);\n  callerSignal?.addEventListener('abort', relayAbort, { once: true });\n  if (callerSignal?.aborted) relayAbort();\n  const timer = setTimeout(() => controller.abort(timeoutReason), timeoutMs);\n  return {\n    controller,\n    timeoutReason,\n    cleanup: () => {\n      clearTimeout(timer);\n      callerSignal?.removeEventListener('abort', relayAbort);\n    },\n  };\n}\n\nfunction disposeResponse(response: Response, controller: AbortController): void {\n  const cancellation = response.body && !response.bodyUsed ? response.body.cancel() : undefined;\n  controller.abort();\n  // The controller owns socket cleanup; observe stream cancellation without delaying the attempt.\n  if (cancellation)\n    void cancellation.then(\n      () => undefined,\n      () => undefined\n    );\n}\n\nfunction sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n  signal?.throwIfAborted();\n  if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));\n  return new Promise((resolve, reject) => {\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const cleanup = () => {\n      if (timer) clearTimeout(timer);\n      signal.removeEventListener('abort', onAbort);\n    };\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    timer = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n    if (signal.aborted) onAbort();\n  });\n}\n\nfunction failedRequest(url: string, phase: ResponsePhase, attempts: number, timeoutMs: number, timedOut: boolean): SenseError {\n  const endpoint = safeUrl(url);\n  if (timedOut) {\n    return new SenseError('EMBED_MODEL', `${endpoint} timed out after ${timeoutMs}ms while waiting for ${phase} (attempt ${attempts}/${MAX_ATTEMPTS}); check the embedding service and network`);\n  }\n  return new SenseError('EMBED_MODEL', `${endpoint} failed while waiting for ${phase} after ${attempts} attempts; check the embedding service and network`);\n}\n\nfunction buffered(response: Response, body: ArrayBuffer, requestUrl: string): BufferedResponse {\n  return {\n    ok: response.ok,\n    status: response.status,\n    headers: response.headers,\n    body,\n    url: safeUrl(response.url || requestUrl),\n  };\n}\n\n// Each attempt covers response headers and the complete successful body. Retry cleanup and\n// backoff honor one caller signal without transferring ownership of that signal.\nexport async function fetchWithRetry(url: string, init: FetchRequestInit, options: FetchRetryOptions = {}): Promise<BufferedResponse> {\n  const timeoutMs = options.attemptTimeoutMs ?? HTTP_ATTEMPT_TIMEOUT_MS;\n  const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    options.signal?.throwIfAborted();\n    const { controller, timeoutReason, cleanup } = attemptController(options.signal, timeoutMs);\n    let response: Response | undefined;\n    let phase: ResponsePhase = 'response headers';\n    let retryDelayMs: number | undefined;\n    try {\n      response = await fetch(url, { ...init, signal: controller.signal });\n      if (!response.ok) {\n        const retryable = response.status === 429 || response.status >= 500;\n        if (retryable && attempt < MAX_ATTEMPTS - 1) {\n          const retryAfter = Number(response.headers.get('retry-after'));\n          retryDelayMs = retryAfter > 0 ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) : baseDelayMs * 2 ** attempt;\n          disposeResponse(response, controller);\n        } else {\n          disposeResponse(response, controller);\n          options.signal?.throwIfAborted();\n          return buffered(response, new ArrayBuffer(0), url);\n        }\n      } else {\n        phase = 'response body';\n        const body = await response.arrayBuffer();\n        options.signal?.throwIfAborted();\n        return buffered(response, body, url);\n      }\n    } catch {\n      try {\n        options.signal?.throwIfAborted();\n      } finally {\n        if (response) disposeResponse(response, controller);\n      }\n      const timedOut = controller.signal.reason === timeoutReason;\n      if (attempt === MAX_ATTEMPTS - 1) throw failedRequest(url, phase, attempt + 1, timeoutMs, timedOut);\n      retryDelayMs = baseDelayMs * 2 ** attempt;\n    } finally {\n      cleanup();\n    }\n    await sleep(retryDelayMs ?? baseDelayMs * 2 ** attempt, options.signal);\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up. Abort it immediately because status does not\n// consume its body, and bound the headers wait for an air-gapped machine.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  const { controller, cleanup } = attemptController(undefined, timeoutMs);\n  try {\n    const response = await fetch(url, { signal: controller.signal });\n    disposeResponse(response, controller);\n    return true;\n  } catch {\n    return false;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["SenseError","MAX_ATTEMPTS","MAX_RETRY_AFTER_MS","DEFAULT_BASE_DELAY_MS","HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","safeUrl","url","parsed","URL","username","password","search","hash","href","attemptController","callerSignal","timeoutMs","throwIfAborted","controller","AbortController","timeoutReason","relayAbort","abort","reason","addEventListener","once","aborted","timer","setTimeout","cleanup","clearTimeout","removeEventListener","disposeResponse","response","cancellation","body","bodyUsed","cancel","undefined","then","sleep","ms","signal","Promise","resolve","reject","onAbort","failedRequest","phase","attempts","timedOut","endpoint","buffered","requestUrl","ok","status","headers","fetchWithRetry","init","options","attemptTimeoutMs","baseDelayMs","attempt","retryDelayMs","fetch","retryable","retryAfter","Number","get","Math","min","ArrayBuffer","arrayBuffer","Error","probeReachable"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAE1C,MAAMC,eAAe;AACrB,MAAMC,qBAAqB;AAC3B,MAAMC,wBAAwB;AAE9B,2FAA2F;AAC3F,OAAO,MAAMC,0BAA0B,MAAO;AAC9C,OAAO,MAAMC,gCAAgC,IAAI,MAAO;AAoBxD,SAASC,QAAQC,GAAW;IAC1B,IAAI;QACF,MAAMC,SAAS,IAAIC,IAAIF;QACvBC,OAAOE,QAAQ,GAAG;QAClBF,OAAOG,QAAQ,GAAG;QAClBH,OAAOI,MAAM,GAAG;QAChBJ,OAAOK,IAAI,GAAG;QACd,OAAOL,OAAOM,IAAI;IACpB,EAAE,OAAM;QACN,OAAO;IACT;AACF;AAEA,SAASC,kBACPC,YAAqC,EACrCC,SAAiB;IAMjBD,yBAAAA,mCAAAA,aAAcE,cAAc;IAC5B,MAAMC,aAAa,IAAIC;IACvB,MAAMC,gBAAgB,CAAC;IACvB,MAAMC,aAAa,IAAMH,WAAWI,KAAK,CAACP,yBAAAA,mCAAAA,aAAcQ,MAAM;IAC9DR,yBAAAA,mCAAAA,aAAcS,gBAAgB,CAAC,SAASH,YAAY;QAAEI,MAAM;IAAK;IACjE,IAAIV,yBAAAA,mCAAAA,aAAcW,OAAO,EAAEL;IAC3B,MAAMM,QAAQC,WAAW,IAAMV,WAAWI,KAAK,CAACF,gBAAgBJ;IAChE,OAAO;QACLE;QACAE;QACAS,SAAS;YACPC,aAAaH;YACbZ,yBAAAA,mCAAAA,aAAcgB,mBAAmB,CAAC,SAASV;QAC7C;IACF;AACF;AAEA,SAASW,gBAAgBC,QAAkB,EAAEf,UAA2B;IACtE,MAAMgB,eAAeD,SAASE,IAAI,IAAI,CAACF,SAASG,QAAQ,GAAGH,SAASE,IAAI,CAACE,MAAM,KAAKC;IACpFpB,WAAWI,KAAK;IAChB,gGAAgG;IAChG,IAAIY,cACF,KAAKA,aAAaK,IAAI,CACpB,IAAMD,WACN,IAAMA;AAEZ;AAEA,SAASE,MAAMC,EAAU,EAAEC,MAA+B;IACxDA,mBAAAA,6BAAAA,OAAQzB,cAAc;IACtB,IAAI,CAACyB,QAAQ,OAAO,IAAIC,QAAQ,CAACC,UAAYhB,WAAWgB,SAASH;IACjE,OAAO,IAAIE,QAAQ,CAACC,SAASC;QAC3B,IAAIlB;QACJ,MAAME,UAAU;YACd,IAAIF,OAAOG,aAAaH;YACxBe,OAAOX,mBAAmB,CAAC,SAASe;QACtC;QACA,MAAMA,UAAU;YACdjB;YACAgB,OAAOH,OAAOnB,MAAM;QACtB;QACAmB,OAAOlB,gBAAgB,CAAC,SAASsB,SAAS;YAAErB,MAAM;QAAK;QACvDE,QAAQC,WAAW;YACjBC;YACAe;QACF,GAAGH;QACH,IAAIC,OAAOhB,OAAO,EAAEoB;IACtB;AACF;AAEA,SAASC,cAAczC,GAAW,EAAE0C,KAAoB,EAAEC,QAAgB,EAAEjC,SAAiB,EAAEkC,QAAiB;IAC9G,MAAMC,WAAW9C,QAAQC;IACzB,IAAI4C,UAAU;QACZ,OAAO,IAAInD,WAAW,eAAe,GAAGoD,SAAS,iBAAiB,EAAEnC,UAAU,qBAAqB,EAAEgC,MAAM,UAAU,EAAEC,SAAS,CAAC,EAAEjD,aAAa,0CAA0C,CAAC;IAC7L;IACA,OAAO,IAAID,WAAW,eAAe,GAAGoD,SAAS,0BAA0B,EAAEH,MAAM,OAAO,EAAEC,SAAS,kDAAkD,CAAC;AAC1J;AAEA,SAASG,SAASnB,QAAkB,EAAEE,IAAiB,EAAEkB,UAAkB;IACzE,OAAO;QACLC,IAAIrB,SAASqB,EAAE;QACfC,QAAQtB,SAASsB,MAAM;QACvBC,SAASvB,SAASuB,OAAO;QACzBrB;QACA7B,KAAKD,QAAQ4B,SAAS3B,GAAG,IAAI+C;IAC/B;AACF;AAEA,2FAA2F;AAC3F,iFAAiF;AACjF,OAAO,eAAeI,eAAenD,GAAW,EAAEoD,IAAsB,EAAEC,UAA6B,CAAC,CAAC;QACrFA,2BACEA;IADpB,MAAM3C,aAAY2C,4BAAAA,QAAQC,gBAAgB,cAAxBD,uCAAAA,4BAA4BxD;IAC9C,MAAM0D,eAAcF,uBAAAA,QAAQE,WAAW,cAAnBF,kCAAAA,uBAAuBzD;IAC3C,IAAK,IAAI4D,UAAU,GAAGA,UAAU9D,cAAc8D,UAAW;YACvDH;SAAAA,kBAAAA,QAAQjB,MAAM,cAAdiB,sCAAAA,gBAAgB1C,cAAc;QAC9B,MAAM,EAAEC,UAAU,EAAEE,aAAa,EAAES,OAAO,EAAE,GAAGf,kBAAkB6C,QAAQjB,MAAM,EAAE1B;QACjF,IAAIiB;QACJ,IAAIe,QAAuB;QAC3B,IAAIe;QACJ,IAAI;YACF9B,WAAW,MAAM+B,MAAM1D,KAAK;gBAAE,GAAGoD,IAAI;gBAAEhB,QAAQxB,WAAWwB,MAAM;YAAC;YACjE,IAAI,CAACT,SAASqB,EAAE,EAAE;gBAChB,MAAMW,YAAYhC,SAASsB,MAAM,KAAK,OAAOtB,SAASsB,MAAM,IAAI;gBAChE,IAAIU,aAAaH,UAAU9D,eAAe,GAAG;oBAC3C,MAAMkE,aAAaC,OAAOlC,SAASuB,OAAO,CAACY,GAAG,CAAC;oBAC/CL,eAAeG,aAAa,IAAIG,KAAKC,GAAG,CAACJ,aAAa,MAAMjE,sBAAsB4D,cAAc,KAAKC;oBACrG9B,gBAAgBC,UAAUf;gBAC5B,OAAO;wBAELyC;oBADA3B,gBAAgBC,UAAUf;qBAC1ByC,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;oBAC9B,OAAOmC,SAASnB,UAAU,IAAIsC,YAAY,IAAIjE;gBAChD;YACF,OAAO;oBAGLqD;gBAFAX,QAAQ;gBACR,MAAMb,OAAO,MAAMF,SAASuC,WAAW;iBACvCb,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;gBAC9B,OAAOmC,SAASnB,UAAUE,MAAM7B;YAClC;QACF,EAAE,OAAM;YACN,IAAI;oBACFqD;iBAAAA,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;YAChC,SAAU;gBACR,IAAIgB,UAAUD,gBAAgBC,UAAUf;YAC1C;YACA,MAAMgC,WAAWhC,WAAWwB,MAAM,CAACnB,MAAM,KAAKH;YAC9C,IAAI0C,YAAY9D,eAAe,GAAG,MAAM+C,cAAczC,KAAK0C,OAAOc,UAAU,GAAG9C,WAAWkC;YAC1Fa,eAAeF,cAAc,KAAKC;QACpC,SAAU;YACRjC;QACF;QACA,MAAMW,MAAMuB,yBAAAA,0BAAAA,eAAgBF,cAAc,KAAKC,SAASH,QAAQjB,MAAM;IACxE;IACA,MAAM,IAAI+B,MAAM;AAClB;AAEA,2FAA2F;AAC3F,0EAA0E;AAC1E,OAAO,eAAeC,eAAepE,GAAW,EAAEU,YAAY,IAAI;IAChE,MAAM,EAAEE,UAAU,EAAEW,OAAO,EAAE,GAAGf,kBAAkBwB,WAAWtB;IAC7D,IAAI;QACF,MAAMiB,WAAW,MAAM+B,MAAM1D,KAAK;YAAEoC,QAAQxB,WAAWwB,MAAM;QAAC;QAC9DV,gBAAgBC,UAAUf;QAC1B,OAAO;IACT,EAAE,OAAM;QACN,OAAO;IACT,SAAU;QACRW;IACF;AACF"}

package/dist/esm/embed/openai.js

@@ -1,17 +1,35 @@
 import { SenseError } from '../errors.js';
-import { fetchWithRetry } from './http.js';
+import { effectiveEmbedEndpoint } from './endpoint.js';
+import { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.js';
 const BATCH_CAP = 64;
+function isEmbedding(value) {
+    if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part)=>typeof part === 'number')) return false;
+    return !('index' in value) || value.index === undefined || typeof value.index === 'number';
+}
+function parseBody(body, endpoint) {
+    let decoded;
+    try {
+        decoded = JSON.parse(new TextDecoder().decode(body));
+    } catch  {
+        throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);
+    }
+    if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {
+        throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);
+    }
+    return decoded.data;
+}
 // One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare
 // Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.
 export async function openaiProvider(model, url, keyEnv) {
-    if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
-    const base = url.replace(/\/+$/, '');
+    const base = effectiveEmbedEndpoint('openai', url);
+    if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
     const headers = {
         'content-type': 'application/json'
     };
     const key = keyEnv ? process.env[keyEnv] : undefined;
     if (key) headers.authorization = `Bearer ${key}`;
-    async function post(texts) {
+    async function post(texts, options) {
+        var _options_signal;
         const res = await fetchWithRetry(`${base}/embeddings`, {
             method: 'POST',
             headers,
@@ -19,17 +37,21 @@ export async function openaiProvider(model, url, keyEnv) {
                 model,
                 input: texts
             })
+        }, {
+            signal: options === null || options === void 0 ? void 0 : options.signal,
+            attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS
         });
-        if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);
-        const body = await res.json();
+        options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+        if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);
+        const body = parseBody(res.body, res.url);
         // The schema carries `index` because response order is not promised, and the caller maps
         // vectors back to chunks positionally: sort by it whenever every element has one.
-        const data = body.data.every((d)=>typeof d.index === 'number') ? [
-            ...body.data
-        ].sort((a, b)=>a.index - b.index) : body.data;
+        const data = body.every((d)=>typeof d.index === 'number') ? [
+            ...body
+        ].sort((a, b)=>a.index - b.index) : body;
         // A server that drops an input (empty or over-length strings are the usual cause) would
         // otherwise shift every later vector onto the wrong chunk, silently.
-        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);
+        if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);
         return data.map((d)=>Float32Array.from(d.embedding));
     }
     const dims = (await post([
@@ -42,8 +64,8 @@ export async function openaiProvider(model, url, keyEnv) {
         dims,
         batchCap: BATCH_CAP,
         embedDocuments: post,
-        embedQuery: async (text)=>(await post([
+        embedQuery: async (text, options)=>(await post([
                 text
-            ]))[0]
+            ], options))[0]
     };
 }

package/dist/esm/embed/openai.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["SenseError","fetchWithRetry","BATCH_CAP","openaiProvider","model","url","keyEnv","base","replace","headers","key","process","env","undefined","authorization","post","texts","res","method","body","JSON","stringify","input","ok","status","json","data","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,cAAc,QAAQ,YAAY;AAG3C,MAAMC,YAAY;AAElB,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,IAAI,CAACD,KAAK,MAAM,IAAIL,WAAW,eAAe;IAC9C,MAAMO,OAAOF,IAAIG,OAAO,CAAC,QAAQ;IACjC,MAAMC,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMJ,SAASK,QAAQC,GAAG,CAACN,OAAO,GAAGO;IAC3C,IAAIH,KAAKD,QAAQK,aAAa,GAAG,CAAC,OAAO,EAAEJ,KAAK;IAEhD,eAAeK,KAAKC,KAAe;QACjC,MAAMC,MAAM,MAAMhB,eAAe,GAAGM,KAAK,WAAW,CAAC,EAAE;YAAEW,QAAQ;YAAQT;YAASU,MAAMC,KAAKC,SAAS,CAAC;gBAAEjB;gBAAOkB,OAAON;YAAM;QAAG;QAChI,IAAI,CAACC,IAAIM,EAAE,EAAE,MAAM,IAAIvB,WAAW,eAAe,GAAGO,KAAK,oBAAoB,EAAEU,IAAIO,MAAM,EAAE;QAC3F,MAAML,OAAQ,MAAMF,IAAIQ,IAAI;QAC5B,yFAAyF;QACzF,kFAAkF;QAClF,MAAMC,OAAOP,KAAKO,IAAI,CAACC,KAAK,CAAC,CAACC,IAAM,OAAOA,EAAEC,KAAK,KAAK,YAAY;eAAIV,KAAKO,IAAI;SAAC,CAACI,IAAI,CAAC,CAACC,GAAGC,IAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK,IAAeV,KAAKO,IAAI;QACvJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIA,KAAKO,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAIjC,WAAW,eAAe,GAAGO,KAAK,qBAAqB,EAAEmB,KAAKO,MAAM,CAAC,gBAAgB,EAAEjB,MAAMiB,MAAM,CAAC,OAAO,CAAC;QACxJ,OAAOP,KAAKQ,GAAG,CAAC,CAACN,IAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;IACtD;IAEA,MAAMC,OAAO,AAAC,CAAA,MAAMvB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACkB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEM,IAAI,CAAC,OAAO,EAAEhC,KAAK,CAAC,EAAEH,OAAO;QAAEkC;QAAME,UAAUtC;QAAWuC,gBAAgB1B;QAAM2B,YAAY,OAAOC,OAAS,AAAC,CAAA,MAAM5B,KAAK;gBAAC4B;aAAK,CAAA,CAAE,CAAC,EAAE;IAAC;AAC/I"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\ninterface OpenaiEmbedding {\n  embedding: number[];\n  index?: number;\n}\n\nfunction isEmbedding(value: unknown): value is OpenaiEmbedding {\n  if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part) => typeof part === 'number')) return false;\n  return !('index' in value) || value.index === undefined || typeof value.index === 'number';\n}\n\nfunction parseBody(body: ArrayBuffer, endpoint: string): OpenaiEmbedding[] {\n  let decoded: unknown;\n  try {\n    decoded = JSON.parse(new TextDecoder().decode(body));\n  } catch {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);\n  }\n  if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);\n  }\n  return decoded.data;\n}\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = effectiveEmbedEndpoint('openai', url);\n  if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) }, { signal: options?.signal, attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n    options?.signal?.throwIfAborted();\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n    const body = parseBody(res.body, res.url);\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.every((d): d is OpenaiEmbedding & { index: number } => typeof d.index === 'number') ? [...body].sort((a, b) => a.index - b.index) : body;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text, options) => (await post([text], options))[0] };\n}\n"],"names":["SenseError","effectiveEmbedEndpoint","fetchWithRetry","HTTP_ATTEMPT_TIMEOUT_MS","BATCH_CAP","isEmbedding","value","Array","isArray","embedding","every","part","index","undefined","parseBody","body","endpoint","decoded","JSON","parse","TextDecoder","decode","data","openaiProvider","model","url","keyEnv","base","headers","key","process","env","authorization","post","texts","options","res","method","stringify","input","signal","attemptTimeoutMs","throwIfAborted","ok","status","d","sort","a","b","length","map","Float32Array","from","dims","id","batchCap","embedDocuments","embedQuery","text"],"mappings":"AAAA,SAASA,UAAU,QAAQ,eAAe;AAC1C,SAASC,sBAAsB,QAAQ,gBAAgB;AACvD,SAASC,cAAc,EAAEC,uBAAuB,QAAQ,YAAY;AAGpE,MAAMC,YAAY;AAOlB,SAASC,YAAYC,KAAc;IACjC,IAAIA,UAAU,QAAQ,OAAOA,UAAU,YAAY,CAAE,CAAA,eAAeA,KAAI,KAAM,CAACC,MAAMC,OAAO,CAACF,MAAMG,SAAS,KAAK,CAACH,MAAMG,SAAS,CAACC,KAAK,CAAC,CAACC,OAAS,OAAOA,SAAS,WAAW,OAAO;IACpL,OAAO,CAAE,CAAA,WAAWL,KAAI,KAAMA,MAAMM,KAAK,KAAKC,aAAa,OAAOP,MAAMM,KAAK,KAAK;AACpF;AAEA,SAASE,UAAUC,IAAiB,EAAEC,QAAgB;IACpD,IAAIC;IACJ,IAAI;QACFA,UAAUC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAACN;IAChD,EAAE,OAAM;QACN,MAAM,IAAIf,WAAW,eAAe,GAAGgB,SAAS,sBAAsB,CAAC;IACzE;IACA,IAAIC,YAAY,QAAQ,OAAOA,YAAY,YAAY,CAAE,CAAA,UAAUA,OAAM,KAAM,CAACV,MAAMC,OAAO,CAACS,QAAQK,IAAI,KAAK,CAACL,QAAQK,IAAI,CAACZ,KAAK,CAACL,cAAc;QAC/I,MAAM,IAAIL,WAAW,eAAe,GAAGgB,SAAS,uCAAuC,CAAC;IAC1F;IACA,OAAOC,QAAQK,IAAI;AACrB;AAEA,6FAA6F;AAC7F,+EAA+E;AAC/E,OAAO,eAAeC,eAAeC,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;IACrG,MAAMC,OAAO1B,uBAAuB,UAAUwB;IAC9C,IAAI,CAACE,MAAM,MAAM,IAAI3B,WAAW,eAAe;IAC/C,MAAM4B,UAAkC;QAAE,gBAAgB;IAAmB;IAC7E,MAAMC,MAAMH,SAASI,QAAQC,GAAG,CAACL,OAAO,GAAGb;IAC3C,IAAIgB,KAAKD,QAAQI,aAAa,GAAG,CAAC,OAAO,EAAEH,KAAK;IAEhD,eAAeI,KAAKC,KAAe,EAAEC,OAA0B;YAE7DA;QADA,MAAMC,MAAM,MAAMlC,eAAe,GAAGyB,KAAK,WAAW,CAAC,EAAE;YAAEU,QAAQ;YAAQT;YAASb,MAAMG,KAAKoB,SAAS,CAAC;gBAAEd;gBAAOe,OAAOL;YAAM;QAAG,GAAG;YAAEM,MAAM,EAAEL,oBAAAA,8BAAAA,QAASK,MAAM;YAAEC,kBAAkBtC;QAAwB;QACxMgC,oBAAAA,+BAAAA,kBAAAA,QAASK,MAAM,cAAfL,sCAAAA,gBAAiBO,cAAc;QAC/B,IAAI,CAACN,IAAIO,EAAE,EAAE,MAAM,IAAI3C,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,SAAS,EAAEW,IAAIQ,MAAM,EAAE;QACnF,MAAM7B,OAAOD,UAAUsB,IAAIrB,IAAI,EAAEqB,IAAIX,GAAG;QACxC,yFAAyF;QACzF,kFAAkF;QAClF,MAAMH,OAAOP,KAAKL,KAAK,CAAC,CAACmC,IAAgD,OAAOA,EAAEjC,KAAK,KAAK,YAAY;eAAIG;SAAK,CAAC+B,IAAI,CAAC,CAACC,GAAGC,IAAMD,EAAEnC,KAAK,GAAGoC,EAAEpC,KAAK,IAAIG;QACtJ,wFAAwF;QACxF,qEAAqE;QACrE,IAAIO,KAAK2B,MAAM,KAAKf,MAAMe,MAAM,EAAE,MAAM,IAAIjD,WAAW,eAAe,GAAGoC,IAAIX,GAAG,CAAC,UAAU,EAAEH,KAAK2B,MAAM,CAAC,gBAAgB,EAAEf,MAAMe,MAAM,CAAC,OAAO,CAAC;QAChJ,OAAO3B,KAAK4B,GAAG,CAAC,CAACL,IAAMM,aAAaC,IAAI,CAACP,EAAEpC,SAAS;IACtD;IAEA,MAAM4C,OAAO,AAAC,CAAA,MAAMpB,KAAK;QAAC;KAAkB,CAAA,CAAE,CAAC,EAAE,CAACgB,MAAM;IACxD,oFAAoF;IACpF,gDAAgD;IAChD,OAAO;QAAEK,IAAI,CAAC,OAAO,EAAE3B,KAAK,CAAC,EAAEH,OAAO;QAAE6B;QAAME,UAAUnD;QAAWoD,gBAAgBvB;QAAMwB,YAAY,OAAOC,MAAMvB,UAAY,AAAC,CAAA,MAAMF,KAAK;gBAACyB;aAAK,EAAEvB,QAAO,CAAE,CAAC,EAAE;IAAC;AACjK"}

package/dist/esm/embed/query.d.ts

@@ -1,12 +1,18 @@
-import type { Config } from '../config/index.js';
+import { type Config } from '../config/index.js';
 import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
+import { type EmbedCallOptions } from './types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options?: EmbedCallOptions): Promise<void>;
 export declare function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void>;
-export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export interface PreparedSemanticQuery {
+    readonly vector: Float32Array;
+    readonly storeDims: number;
+}
+export declare function prepareSemanticQuery(cfg: Config, terms: string, options?: EmbedCallOptions): Promise<PreparedSemanticQuery>;
+export declare function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
 export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;

package/dist/esm/embed/query.js

@@ -1,8 +1,13 @@
+import { featureSignature } from '../config/index.js';
 import { SenseError } from '../errors.js';
 import { embed } from '../features/embed.js';
+import { FEATURES } from '../features/index.js';
 import { progress } from '../output/progress.js';
 import { parseSource } from '../scan/index.js';
-import { takeChunkText } from './handoff.js';
+import { FEATURE_SIGNATURE_META_KEY } from '../store/readiness.js';
+import { getMeta } from '../store/shared.js';
+import { embeddingSignatureCompatible } from '../store/signature.js';
+import { identifyChunkText, takeChunkText } from './handoff.js';
 import { checkLanguageFit } from './langfit.js';
 import { getProvider } from './registry.js';
 import { STORE_DIMS } from './types.js';
@@ -28,38 +33,64 @@ export function toStore(full, dims, int8) {
 // Embed rows whose vector is NULL. Reconcile hands its chunk text over in memory when it ran in
 // this process (handoff.ts); a row left pending by an earlier command re-derives from the exact
 // source string persisted with that indexed generation.
-export async function embedPending(store, cfg, allowed) {
+function staleEmbeddingWork(reason) {
+    return new SenseError('INDEX_NOT_READY', `${reason}; run \`sense build\` (or library build(config)) again to prepare the current index generation`);
+}
+// expectedSignature is captured by open() before provider work. Direct internal callers omit it
+// and capture the durable value here, before selecting chunks or constructing a provider.
+export async function embedPending(store, cfg, allowed, expectedSignature, options = {}) {
     var _takeChunkText;
-    const provider = await getProvider(cfg); // throws EMBED_DISABLED before touching the table
-    const dirty = (await store.vectors.pending()).filter((row)=>{
+    var _options_signal, _options_signal1, _options_signal2, _options_signal3, // Over the exact texts about to be embedded, before any of them are: a mismatch fails
+    // this run loudly instead of quietly storing vectors from the wrong model.
+    _options_signal4, _options_signal5;
+    (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+    const expectedDurableSignature = expectedSignature === undefined ? await getMeta(store, FEATURE_SIGNATURE_META_KEY) : expectedSignature;
+    (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+    if (!embeddingSignatureCompatible(expectedDurableSignature, featureSignature(cfg, FEATURES))) {
+        throw staleEmbeddingWork('the index configuration changed before vectors could be prepared');
+    }
+    const pendingStmt = await store.prepare('SELECT "path", chunk, content_identity FROM embeddings WHERE vector IS NULL ORDER BY "path", chunk');
+    const dirty = (await pendingStmt.all()).filter((row)=>{
         var _ref;
         return (_ref = allowed === null || allowed === void 0 ? void 0 : allowed.has(row.path)) !== null && _ref !== void 0 ? _ref : true;
     });
+    (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
     if (dirty.length === 0) return;
+    const provider = await getProvider(cfg, options);
+    (_options_signal3 = options.signal) === null || _options_signal3 === void 0 ? void 0 : _options_signal3.throwIfAborted();
+    const submittedSignature = featureSignature(cfg, FEATURES);
+    if (!embeddingSignatureCompatible(expectedDurableSignature, submittedSignature)) {
+        throw staleEmbeddingWork('the index configuration changed while the embedding provider was being prepared');
+    }
     const storeDims = Math.min(STORE_DIMS, provider.dims);
     const byPath = new Map();
     for (const row of dirty){
         var _byPath_get;
         const list = (_byPath_get = byPath.get(row.path)) !== null && _byPath_get !== void 0 ? _byPath_get : [];
-        list.push(row.chunk);
+        list.push(row);
         byPath.set(row.path, list);
     }
     const textByPath = (_takeChunkText = takeChunkText(store)) !== null && _takeChunkText !== void 0 ? _takeChunkText : new Map();
     const sourceStmt = await store.prepare('SELECT text FROM indexed_sources WHERE "path" = ?');
     const jobs = [];
-    for (const [path, chunkIdxs] of byPath){
+    for (const [path, pending] of byPath){
+        var _options_signal6;
         const texts = textByPath.get(path);
-        if (texts && chunkIdxs.every((idx)=>idx < texts.length)) {
-            for (const idx of chunkIdxs)jobs.push({
+        if (texts && pending.every((row)=>{
+            var _texts_row_chunk;
+            return ((_texts_row_chunk = texts[row.chunk]) === null || _texts_row_chunk === void 0 ? void 0 : _texts_row_chunk.identity) === row.content_identity;
+        })) {
+            for (const row of pending)jobs.push({
                 path,
-                chunk: idx,
-                text: texts[idx]
+                chunk: row.chunk,
+                ...texts[row.chunk]
             });
             continue;
         }
         // A prior process may have reconciled the row, so its in-memory handoff is gone. Re-derive
         // from that indexed generation's stored source, never from a newer live file.
         const source = await sourceStmt.get(path);
+        (_options_signal6 = options.signal) === null || _options_signal6 === void 0 ? void 0 : _options_signal6.throwIfAborted();
         if (!source) throw new SenseError('INDEX_NOT_READY', `indexed source text is missing for ${path}; run \`sense build --force\` (or library build(config, { force: true })) before preparing vectors`);
         const chunks = parseSource({
             relPath: path,
@@ -72,26 +103,33 @@ export async function embedPending(store, cfg, allowed) {
         }, source.text, [
             embed
         ], cfg).doc.extracted.embed;
-        for (const idx of chunkIdxs){
-            const chunk = chunks[idx];
+        for (const row of pending){
+            const chunk = chunks[row.chunk];
             if (!chunk) throw new SenseError('INDEX_NOT_READY', `indexed vector metadata for ${path} does not match its indexed source; run \`sense build --force\` (or library build(config, { force: true }))`);
+            const identified = identifyChunkText(chunk.text);
+            if (identified.identity !== row.content_identity) throw staleEmbeddingWork(`indexed chunk ${path}#${row.chunk} changed before vectors could be prepared`);
             jobs.push({
                 path,
-                chunk: idx,
-                text: chunk.text
+                chunk: row.chunk,
+                ...identified
             });
         }
     }
-    // Over the exact texts about to be embedded, before any of them are: a mismatch fails
-    // this run loudly instead of quietly storing vectors from the wrong model.
+    (_options_signal4 = options.signal) === null || _options_signal4 === void 0 ? void 0 : _options_signal4.throwIfAborted();
     await checkLanguageFit(store, provider, jobs.map((j)=>j.text));
+    (_options_signal5 = options.signal) === null || _options_signal5 === void 0 ? void 0 : _options_signal5.throwIfAborted();
     // Preparing a large configured scope can take long enough to look stalled, so report each
     // completed provider batch.
     const report = progress('embedding chunks', jobs.length);
     for(let i = 0; i < jobs.length; i += provider.batchCap){
+        var _options_signal7, _options_signal8, _options_signal9;
         const batch = jobs.slice(i, i + provider.batchCap);
-        const vectors = await provider.embedDocuments(batch.map((j)=>j.text));
-        await store.vectors.writeVectors(batch.map((job, j)=>{
+        (_options_signal7 = options.signal) === null || _options_signal7 === void 0 ? void 0 : _options_signal7.throwIfAborted();
+        if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed before a provider batch was submitted');
+        const vectors = await provider.embedDocuments(batch.map((j)=>j.text), options);
+        (_options_signal8 = options.signal) === null || _options_signal8 === void 0 ? void 0 : _options_signal8.throwIfAborted();
+        if (featureSignature(cfg, FEATURES) !== submittedSignature) throw staleEmbeddingWork('the embedding configuration changed while a provider batch was running');
+        const writes = batch.map((job, j)=>{
             const { v, scale } = toStore(vectors[j], storeDims, true);
             const q = new Int8Array(storeDims);
             for(let d = 0; d < storeDims; d++)q[d] = Math.round(v[d] / scale);
@@ -101,7 +139,28 @@ export async function embedPending(store, cfg, allowed) {
                 scale,
                 vector: Buffer.from(q.buffer)
             };
-        }));
+        });
+        (_options_signal9 = options.signal) === null || _options_signal9 === void 0 ? void 0 : _options_signal9.throwIfAborted();
+        await store.transaction(async ()=>{
+            var // Write first so SQLite owns the write lock before the fence reads. Any mismatch throws
+            // inside this transaction and rolls every row in the provider batch back together.
+            _options_signal, _options_signal1, _options_signal2;
+            (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+            await store.vectors.writeVectors(writes);
+            (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+            if (await getMeta(store, FEATURE_SIGNATURE_META_KEY) !== expectedDurableSignature) throw staleEmbeddingWork('the durable embedding configuration changed while a provider batch was running');
+            const currentStmt = await store.prepare(`SELECT "path", chunk, content_identity FROM embeddings WHERE ${batch.map(()=>'("path" = ? AND chunk = ?)').join(' OR ')}`);
+            const current = await currentStmt.all(...batch.flatMap((job)=>[
+                    job.path,
+                    job.chunk
+                ]));
+            const identities = new Map(current.map((row)=>[
+                    `${row.path}\0${Number(row.chunk)}`,
+                    row.content_identity
+                ]));
+            if (batch.some((job)=>identities.get(`${job.path}\0${job.chunk}`) !== job.identity)) throw staleEmbeddingWork('indexed chunks changed while a provider batch was running');
+            (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
+        });
         report.tick(Math.min(i + provider.batchCap, jobs.length));
     }
     report.finish();
@@ -112,22 +171,35 @@ export async function ensureDocumentEmbeddings(store, _cfg, allowed) {
         throw new SenseError('INDEX_NOT_READY', `${pending.length} indexed chunk(s) in the requested scope still need vectors; run \`sense build\` (or library build(config)) before querying with build disabled`);
     }
 }
-// Best chunk per file by cosine, its line range riding along; FTS5 operators are stripped as
-// lexical syntax. Similarity comes back because the fused score cannot express match quality.
-export async function semanticCandidates(store, cfg, terms, fetch, allowed) {
+// Provider/model work is independent of the index snapshot. FTS5 operators are stripped as
+// lexical syntax before the query is embedded, matching the prior semantic query behavior.
+export async function prepareSemanticQuery(cfg, terms, options = {}) {
     var _cfg_rootDir, _terms_match;
+    var _options_signal, _options_signal1, _options_signal2;
     const rootDir = (_cfg_rootDir = cfg.rootDir) !== null && _cfg_rootDir !== void 0 ? _cfg_rootDir : cfg.baseDir;
     if (!rootDir) throw new SenseError('EMBED_MODEL', 'semantic expansion needs a resolved config (use loadConfig/open)');
-    const provider = await getProvider(cfg);
-    const storeDims = Math.min(STORE_DIMS, provider.dims);
     const text = ((_terms_match = terms.match(/[\p{L}\p{N}]+/gu)) !== null && _terms_match !== void 0 ? _terms_match : []).filter((t)=>![
             'AND',
             'OR',
             'NOT',
             'NEAR'
         ].includes(t)).join(' ');
-    const { v: qv } = toStore(await provider.embedQuery(text), storeDims, false);
-    return store.vectors.candidates(qv, storeDims, fetch, allowed);
+    (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+    const provider = await getProvider(cfg, options);
+    (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+    const storeDims = Math.min(STORE_DIMS, provider.dims);
+    const queryVector = await provider.embedQuery(text, options);
+    (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
+    const { v: qv } = toStore(queryVector, storeDims, false);
+    return {
+        vector: qv,
+        storeDims
+    };
+}
+// Best chunk per file by cosine, its line range riding along. The caller owns the read snapshot
+// and the prepared vector, so this performs no provider work and only delegates to the store.
+export function semanticCandidates(store, prepared, fetch, allowed) {
+    return store.vectors.candidates(prepared.vector, prepared.storeDims, fetch, allowed);
 }
 // Whether a note has any chunk with a vector. Distinguishes "nothing is near this note" from
 // "this note has no text", which look the same in an empty result.