sensemaking 0.25.1 → 0.27.0

package/dist/cjs/embed/http.d.ts

@@ -1,2 +1,19 @@
-export declare function fetchWithRetry(url: string, init: RequestInit, baseDelayMs?: number): Promise<Response>;
+export declare const HTTP_ATTEMPT_TIMEOUT_MS = 30000;
+export declare const MODEL_FILE_ATTEMPT_TIMEOUT_MS: number;
+export type FetchRequestInit = Omit<RequestInit, 'signal'> & {
+    signal?: never;
+};
+export interface FetchRetryOptions {
+    signal?: AbortSignal;
+    attemptTimeoutMs?: number;
+    baseDelayMs?: number;
+}
+export interface BufferedResponse {
+    readonly ok: boolean;
+    readonly status: number;
+    readonly headers: Headers;
+    readonly body: ArrayBuffer;
+    readonly url: string;
+}
+export declare function fetchWithRetry(url: string, init: FetchRequestInit, options?: FetchRetryOptions): Promise<BufferedResponse>;
 export declare function probeReachable(url: string, timeoutMs?: number): Promise<boolean>;

package/dist/cjs/embed/http.js

@@ -9,6 +9,12 @@ function _export(target, all) {
     });
 }
 _export(exports, {
+    get HTTP_ATTEMPT_TIMEOUT_MS () {
+        return HTTP_ATTEMPT_TIMEOUT_MS;
+    },
+    get MODEL_FILE_ATTEMPT_TIMEOUT_MS () {
+        return MODEL_FILE_ATTEMPT_TIMEOUT_MS;
+    },
     get fetchWithRetry () {
         return fetchWithRetry;
     },
@@ -16,6 +22,7 @@ _export(exports, {
         return probeReachable;
     }
 });
+var _errorsts = require("../errors.js");
 function asyncGeneratorStep(gen, resolve, reject, _next, _throw, key, arg) {
     try {
         var info = gen[key](arg);
@@ -42,6 +49,55 @@ function _async_to_generator(fn) {
         });
     };
 }
+function _define_property(obj, key, value) {
+    if (key in obj) {
+        Object.defineProperty(obj, key, {
+            value: value,
+            enumerable: true,
+            configurable: true,
+            writable: true
+        });
+    } else obj[key] = value;
+    return obj;
+}
+function _object_spread(target) {
+    for(var i = 1; i < arguments.length; i++){
+        var source = arguments[i] != null ? arguments[i] : {};
+        var ownKeys = Object.keys(source);
+        if (typeof Object.getOwnPropertySymbols === "function") {
+            ownKeys = ownKeys.concat(Object.getOwnPropertySymbols(source).filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(source, sym).enumerable;
+            }));
+        }
+        ownKeys.forEach(function(key) {
+            _define_property(target, key, source[key]);
+        });
+    }
+    return target;
+}
+function ownKeys(object, enumerableOnly) {
+    var keys = Object.keys(object);
+    if (Object.getOwnPropertySymbols) {
+        var symbols = Object.getOwnPropertySymbols(object);
+        if (enumerableOnly) {
+            symbols = symbols.filter(function(sym) {
+                return Object.getOwnPropertyDescriptor(object, sym).enumerable;
+            });
+        }
+        keys.push.apply(keys, symbols);
+    }
+    return keys;
+}
+function _object_spread_props(target, source) {
+    source = source != null ? source : {};
+    if (Object.getOwnPropertyDescriptors) Object.defineProperties(target, Object.getOwnPropertyDescriptors(source));
+    else {
+        ownKeys(Object(source)).forEach(function(key) {
+            Object.defineProperty(target, key, Object.getOwnPropertyDescriptor(source, key));
+        });
+    }
+    return target;
+}
 function _ts_generator(thisArg, body) {
     var f, y, t, _ = {
         label: 0,
@@ -142,112 +198,255 @@ function _ts_generator(thisArg, body) {
     }
 }
 var MAX_ATTEMPTS = 3;
-function sleep(ms) {
-    return new Promise(function(resolve) {
+var MAX_RETRY_AFTER_MS = 10000;
+var DEFAULT_BASE_DELAY_MS = 200;
+var HTTP_ATTEMPT_TIMEOUT_MS = 30000;
+var MODEL_FILE_ATTEMPT_TIMEOUT_MS = 5 * 60000;
+function safeUrl(url) {
+    try {
+        var parsed = new URL(url);
+        parsed.username = '';
+        parsed.password = '';
+        parsed.search = '';
+        parsed.hash = '';
+        return parsed.href;
+    } catch (unused) {
+        return 'the configured embedding endpoint';
+    }
+}
+function attemptController(callerSignal, timeoutMs) {
+    callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.throwIfAborted();
+    var controller = new AbortController();
+    var timeoutReason = {};
+    var relayAbort = function relayAbort() {
+        return controller.abort(callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.reason);
+    };
+    callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.addEventListener('abort', relayAbort, {
+        once: true
+    });
+    if (callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.aborted) relayAbort();
+    var timer = setTimeout(function() {
+        return controller.abort(timeoutReason);
+    }, timeoutMs);
+    return {
+        controller: controller,
+        timeoutReason: timeoutReason,
+        cleanup: function cleanup() {
+            clearTimeout(timer);
+            callerSignal === null || callerSignal === void 0 ? void 0 : callerSignal.removeEventListener('abort', relayAbort);
+        }
+    };
+}
+function disposeResponse(response, controller) {
+    var cancellation = response.body && !response.bodyUsed ? response.body.cancel() : undefined;
+    controller.abort();
+    // The controller owns socket cleanup; observe stream cancellation without delaying the attempt.
+    if (cancellation) void cancellation.then(function() {
+        return undefined;
+    }, function() {
+        return undefined;
+    });
+}
+function sleep(ms, signal) {
+    signal === null || signal === void 0 ? void 0 : signal.throwIfAborted();
+    if (!signal) return new Promise(function(resolve) {
         return setTimeout(resolve, ms);
     });
+    return new Promise(function(resolve, reject) {
+        var timer;
+        var cleanup = function cleanup() {
+            if (timer) clearTimeout(timer);
+            signal.removeEventListener('abort', onAbort);
+        };
+        var onAbort = function onAbort() {
+            cleanup();
+            reject(signal.reason);
+        };
+        signal.addEventListener('abort', onAbort, {
+            once: true
+        });
+        timer = setTimeout(function() {
+            cleanup();
+            resolve();
+        }, ms);
+        if (signal.aborted) onAbort();
+    });
 }
-function fetchWithRetry(url, init) {
-    var baseDelayMs = arguments.length > 2 && arguments[2] !== void 0 ? arguments[2] : 200;
-    return _async_to_generator(function() {
-        var attempt, res, retryAfter, err;
+function failedRequest(url, phase, attempts, timeoutMs, timedOut) {
+    var endpoint = safeUrl(url);
+    if (timedOut) {
+        return new _errorsts.SenseError('EMBED_MODEL', "".concat(endpoint, " timed out after ").concat(timeoutMs, "ms while waiting for ").concat(phase, " (attempt ").concat(attempts, "/").concat(MAX_ATTEMPTS, "); check the embedding service and network"));
+    }
+    return new _errorsts.SenseError('EMBED_MODEL', "".concat(endpoint, " failed while waiting for ").concat(phase, " after ").concat(attempts, " attempts; check the embedding service and network"));
+}
+function buffered(response, body, requestUrl) {
+    return {
+        ok: response.ok,
+        status: response.status,
+        headers: response.headers,
+        body: body,
+        url: safeUrl(response.url || requestUrl)
+    };
+}
+function fetchWithRetry(_0, _1) {
+    return _async_to_generator(function(url, init) {
+        var options, _options_attemptTimeoutMs, _options_baseDelayMs, timeoutMs, baseDelayMs, attempt, _options_signal, _attemptController, controller, timeoutReason, cleanup, response, phase, retryDelayMs, retryable, retryAfter, _options_signal1, _options_signal2, body, unused, _options_signal3, timedOut;
+        var _arguments = arguments;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
+                    options = _arguments.length > 2 && _arguments[2] !== void 0 ? _arguments[2] : {};
+                    timeoutMs = (_options_attemptTimeoutMs = options.attemptTimeoutMs) !== null && _options_attemptTimeoutMs !== void 0 ? _options_attemptTimeoutMs : HTTP_ATTEMPT_TIMEOUT_MS;
+                    baseDelayMs = (_options_baseDelayMs = options.baseDelayMs) !== null && _options_baseDelayMs !== void 0 ? _options_baseDelayMs : DEFAULT_BASE_DELAY_MS;
                     attempt = 0;
                     _state.label = 1;
                 case 1:
                     if (!(attempt < MAX_ATTEMPTS)) return [
                         3,
-                        8
+                        12
                     ];
+                    (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+                    _attemptController = attemptController(options.signal, timeoutMs), controller = _attemptController.controller, timeoutReason = _attemptController.timeoutReason, cleanup = _attemptController.cleanup;
+                    response = void 0;
+                    phase = 'response headers';
+                    retryDelayMs = void 0;
                     _state.label = 2;
                 case 2:
                     _state.trys.push([
                         2,
-                        5,
-                        ,
-                        7
+                        7,
+                        8,
+                        9
                     ]);
                     return [
                         4,
-                        fetch(url, init)
+                        fetch(url, _object_spread_props(_object_spread({}, init), {
+                            signal: controller.signal
+                        }))
                     ];
                 case 3:
-                    res = _state.sent();
-                    if (res.ok || res.status !== 429 && res.status < 500 || attempt === MAX_ATTEMPTS - 1) return [
-                        2,
-                        res
+                    response = _state.sent();
+                    if (!!response.ok) return [
+                        3,
+                        4
                     ];
-                    retryAfter = Number(res.headers.get('retry-after'));
-                    // Capped: a server naming a huge Retry-After must not stall an embed run for it.
+                    retryable = response.status === 429 || response.status >= 500;
+                    if (retryable && attempt < MAX_ATTEMPTS - 1) {
+                        retryAfter = Number(response.headers.get('retry-after'));
+                        retryDelayMs = retryAfter > 0 ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) : baseDelayMs * Math.pow(2, attempt);
+                        disposeResponse(response, controller);
+                    } else {
+                        ;
+                        disposeResponse(response, controller);
+                        (_options_signal1 = options.signal) === null || _options_signal1 === void 0 ? void 0 : _options_signal1.throwIfAborted();
+                        return [
+                            2,
+                            buffered(response, new ArrayBuffer(0), url)
+                        ];
+                    }
                     return [
-                        4,
-                        sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10000) : baseDelayMs * Math.pow(2, attempt))
+                        3,
+                        6
                     ];
                 case 4:
-                    _state.sent();
+                    phase = 'response body';
                     return [
-                        3,
-                        7
+                        4,
+                        response.arrayBuffer()
                     ];
                 case 5:
-                    err = _state.sent();
-                    if (attempt === MAX_ATTEMPTS - 1) throw err;
+                    body = _state.sent();
+                    (_options_signal2 = options.signal) === null || _options_signal2 === void 0 ? void 0 : _options_signal2.throwIfAborted();
                     return [
-                        4,
-                        sleep(baseDelayMs * Math.pow(2, attempt))
+                        2,
+                        buffered(response, body, url)
                     ];
                 case 6:
-                    _state.sent();
                     return [
                         3,
-                        7
+                        9
                     ];
                 case 7:
+                    unused = _state.sent();
+                    try {
+                        ;
+                        (_options_signal3 = options.signal) === null || _options_signal3 === void 0 ? void 0 : _options_signal3.throwIfAborted();
+                    } finally{
+                        if (response) disposeResponse(response, controller);
+                    }
+                    timedOut = controller.signal.reason === timeoutReason;
+                    if (attempt === MAX_ATTEMPTS - 1) throw failedRequest(url, phase, attempt + 1, timeoutMs, timedOut);
+                    retryDelayMs = baseDelayMs * Math.pow(2, attempt);
+                    return [
+                        3,
+                        9
+                    ];
+                case 8:
+                    cleanup();
+                    return [
+                        7
+                    ];
+                case 9:
+                    return [
+                        4,
+                        sleep(retryDelayMs !== null && retryDelayMs !== void 0 ? retryDelayMs : baseDelayMs * Math.pow(2, attempt), options.signal)
+                    ];
+                case 10:
+                    _state.sent();
+                    _state.label = 11;
+                case 11:
                     attempt++;
                     return [
                         3,
                         1
                     ];
-                case 8:
+                case 12:
                     throw new Error('unreachable');
             }
         });
-    })();
+    }).apply(this, arguments);
 }
 function probeReachable(url) {
     var timeoutMs = arguments.length > 1 && arguments[1] !== void 0 ? arguments[1] : 1500;
     return _async_to_generator(function() {
-        var unused;
+        var _attemptController, controller, cleanup, response, unused;
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
+                    _attemptController = attemptController(undefined, timeoutMs), controller = _attemptController.controller, cleanup = _attemptController.cleanup;
+                    _state.label = 1;
+                case 1:
                     _state.trys.push([
-                        0,
-                        2,
-                        ,
-                        3
+                        1,
+                        3,
+                        4,
+                        5
                     ]);
                     return [
                         4,
                         fetch(url, {
-                            signal: AbortSignal.timeout(timeoutMs)
+                            signal: controller.signal
                         })
                     ];
-                case 1:
-                    _state.sent();
+                case 2:
+                    response = _state.sent();
+                    disposeResponse(response, controller);
                     return [
                         2,
                         true
                     ];
-                case 2:
+                case 3:
                     unused = _state.sent();
                     return [
                         2,
                         false
                     ];
-                case 3:
+                case 4:
+                    cleanup();
+                    return [
+                        7
+                    ];
+                case 5:
                     return [
                         2
                     ];

package/dist/cjs/embed/http.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["const MAX_ATTEMPTS = 3;\n\nfunction sleep(ms: number): Promise<void> {\n  return new Promise((resolve) => setTimeout(resolve, ms));\n}\n\n// Shared retry glue for the HTTP providers: 429/5xx/network failure retry with exponential\n// backoff honoring Retry-After; any other status returns immediately for the caller to handle.\nexport async function fetchWithRetry(url: string, init: RequestInit, baseDelayMs = 200): Promise<Response> {\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    try {\n      const res = await fetch(url, init);\n      if (res.ok || (res.status !== 429 && res.status < 500) || attempt === MAX_ATTEMPTS - 1) return res;\n      const retryAfter = Number(res.headers.get('retry-after'));\n      // Capped: a server naming a huge Retry-After must not stall an embed run for it.\n      await sleep(retryAfter > 0 ? Math.min(retryAfter * 1000, 10_000) : baseDelayMs * 2 ** attempt);\n    } catch (err) {\n      if (attempt === MAX_ATTEMPTS - 1) throw err;\n      await sleep(baseDelayMs * 2 ** attempt);\n    }\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up; only a network failure or the timeout counts as\n// unreachable, so an air-gapped machine never waits more than timeoutMs on `sense status`.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  try {\n    await fetch(url, { signal: AbortSignal.timeout(timeoutMs) });\n    return true;\n  } catch {\n    return false;\n  }\n}\n"],"names":["fetchWithRetry","probeReachable","MAX_ATTEMPTS","sleep","ms","Promise","resolve","setTimeout","url","init","baseDelayMs","attempt","res","retryAfter","err","fetch","ok","status","Number","headers","get","Math","min","Error","timeoutMs","signal","AbortSignal","timeout"],"mappings":";;;;;;;;;;;QAQsBA;eAAAA;;QAkBAC;eAAAA;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AA1BtB,IAAMC,eAAe;AAErB,SAASC,MAAMC,EAAU;IACvB,OAAO,IAAIC,QAAQ,SAACC;eAAYC,WAAWD,SAASF;;AACtD;AAIO,SAAeJ,eAAeQ,GAAW,EAAEC,IAAiB;QAAEC,cAAAA,iEAAc;;YACxEC,SAECC,KAEAC,YAGCC;;;;oBAPFH,UAAU;;;yBAAGA,CAAAA,UAAUT,YAAW;;;;;;;;;;;;oBAE3B;;wBAAMa,MAAMP,KAAKC;;;oBAAvBG,MAAM;oBACZ,IAAIA,IAAII,EAAE,IAAKJ,IAAIK,MAAM,KAAK,OAAOL,IAAIK,MAAM,GAAG,OAAQN,YAAYT,eAAe,GAAG;;wBAAOU;;oBACzFC,aAAaK,OAAON,IAAIO,OAAO,CAACC,GAAG,CAAC;oBAC1C,iFAAiF;oBACjF;;wBAAMjB,MAAMU,aAAa,IAAIQ,KAAKC,GAAG,CAACT,aAAa,MAAM,SAAUH,uBAAc,GAAKC;;;oBAAtF;;;;;;oBACOG;oBACP,IAAIH,YAAYT,eAAe,GAAG,MAAMY;oBACxC;;wBAAMX,MAAMO,uBAAc,GAAKC;;;oBAA/B;;;;;;oBAT0CA;;;;;;oBAY9C,MAAM,IAAIY,MAAM;;;IAClB;;AAIO,SAAetB,eAAeO,GAAW;QAAEgB,YAAAA,iEAAY;;;;;;;;;;;;oBAE1D;;wBAAMT,MAAMP,KAAK;4BAAEiB,QAAQC,YAAYC,OAAO,CAACH;wBAAW;;;oBAA1D;oBACA;;wBAAO;;;;oBAEP;;wBAAO;;;;;;;;IAEX"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/http.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\n\nconst MAX_ATTEMPTS = 3;\nconst MAX_RETRY_AFTER_MS = 10_000;\nconst DEFAULT_BASE_DELAY_MS = 200;\n\n// Initial policy bounds small API/metadata responses separately from model-file downloads.\nexport const HTTP_ATTEMPT_TIMEOUT_MS = 30_000;\nexport const MODEL_FILE_ATTEMPT_TIMEOUT_MS = 5 * 60_000;\n\nexport type FetchRequestInit = Omit<RequestInit, 'signal'> & { signal?: never };\n\nexport interface FetchRetryOptions {\n  signal?: AbortSignal;\n  attemptTimeoutMs?: number;\n  baseDelayMs?: number;\n}\n\nexport interface BufferedResponse {\n  readonly ok: boolean;\n  readonly status: number;\n  readonly headers: Headers;\n  readonly body: ArrayBuffer;\n  readonly url: string;\n}\n\ntype ResponsePhase = 'response headers' | 'response body';\n\nfunction safeUrl(url: string): string {\n  try {\n    const parsed = new URL(url);\n    parsed.username = '';\n    parsed.password = '';\n    parsed.search = '';\n    parsed.hash = '';\n    return parsed.href;\n  } catch {\n    return 'the configured embedding endpoint';\n  }\n}\n\nfunction attemptController(\n  callerSignal: AbortSignal | undefined,\n  timeoutMs: number\n): {\n  controller: AbortController;\n  timeoutReason: object;\n  cleanup: () => void;\n} {\n  callerSignal?.throwIfAborted();\n  const controller = new AbortController();\n  const timeoutReason = {};\n  const relayAbort = () => controller.abort(callerSignal?.reason);\n  callerSignal?.addEventListener('abort', relayAbort, { once: true });\n  if (callerSignal?.aborted) relayAbort();\n  const timer = setTimeout(() => controller.abort(timeoutReason), timeoutMs);\n  return {\n    controller,\n    timeoutReason,\n    cleanup: () => {\n      clearTimeout(timer);\n      callerSignal?.removeEventListener('abort', relayAbort);\n    },\n  };\n}\n\nfunction disposeResponse(response: Response, controller: AbortController): void {\n  const cancellation = response.body && !response.bodyUsed ? response.body.cancel() : undefined;\n  controller.abort();\n  // The controller owns socket cleanup; observe stream cancellation without delaying the attempt.\n  if (cancellation)\n    void cancellation.then(\n      () => undefined,\n      () => undefined\n    );\n}\n\nfunction sleep(ms: number, signal: AbortSignal | undefined): Promise<void> {\n  signal?.throwIfAborted();\n  if (!signal) return new Promise((resolve) => setTimeout(resolve, ms));\n  return new Promise((resolve, reject) => {\n    let timer: ReturnType<typeof setTimeout> | undefined;\n    const cleanup = () => {\n      if (timer) clearTimeout(timer);\n      signal.removeEventListener('abort', onAbort);\n    };\n    const onAbort = () => {\n      cleanup();\n      reject(signal.reason);\n    };\n    signal.addEventListener('abort', onAbort, { once: true });\n    timer = setTimeout(() => {\n      cleanup();\n      resolve();\n    }, ms);\n    if (signal.aborted) onAbort();\n  });\n}\n\nfunction failedRequest(url: string, phase: ResponsePhase, attempts: number, timeoutMs: number, timedOut: boolean): SenseError {\n  const endpoint = safeUrl(url);\n  if (timedOut) {\n    return new SenseError('EMBED_MODEL', `${endpoint} timed out after ${timeoutMs}ms while waiting for ${phase} (attempt ${attempts}/${MAX_ATTEMPTS}); check the embedding service and network`);\n  }\n  return new SenseError('EMBED_MODEL', `${endpoint} failed while waiting for ${phase} after ${attempts} attempts; check the embedding service and network`);\n}\n\nfunction buffered(response: Response, body: ArrayBuffer, requestUrl: string): BufferedResponse {\n  return {\n    ok: response.ok,\n    status: response.status,\n    headers: response.headers,\n    body,\n    url: safeUrl(response.url || requestUrl),\n  };\n}\n\n// Each attempt covers response headers and the complete successful body. Retry cleanup and\n// backoff honor one caller signal without transferring ownership of that signal.\nexport async function fetchWithRetry(url: string, init: FetchRequestInit, options: FetchRetryOptions = {}): Promise<BufferedResponse> {\n  const timeoutMs = options.attemptTimeoutMs ?? HTTP_ATTEMPT_TIMEOUT_MS;\n  const baseDelayMs = options.baseDelayMs ?? DEFAULT_BASE_DELAY_MS;\n  for (let attempt = 0; attempt < MAX_ATTEMPTS; attempt++) {\n    options.signal?.throwIfAborted();\n    const { controller, timeoutReason, cleanup } = attemptController(options.signal, timeoutMs);\n    let response: Response | undefined;\n    let phase: ResponsePhase = 'response headers';\n    let retryDelayMs: number | undefined;\n    try {\n      response = await fetch(url, { ...init, signal: controller.signal });\n      if (!response.ok) {\n        const retryable = response.status === 429 || response.status >= 500;\n        if (retryable && attempt < MAX_ATTEMPTS - 1) {\n          const retryAfter = Number(response.headers.get('retry-after'));\n          retryDelayMs = retryAfter > 0 ? Math.min(retryAfter * 1000, MAX_RETRY_AFTER_MS) : baseDelayMs * 2 ** attempt;\n          disposeResponse(response, controller);\n        } else {\n          disposeResponse(response, controller);\n          options.signal?.throwIfAborted();\n          return buffered(response, new ArrayBuffer(0), url);\n        }\n      } else {\n        phase = 'response body';\n        const body = await response.arrayBuffer();\n        options.signal?.throwIfAborted();\n        return buffered(response, body, url);\n      }\n    } catch {\n      try {\n        options.signal?.throwIfAborted();\n      } finally {\n        if (response) disposeResponse(response, controller);\n      }\n      const timedOut = controller.signal.reason === timeoutReason;\n      if (attempt === MAX_ATTEMPTS - 1) throw failedRequest(url, phase, attempt + 1, timeoutMs, timedOut);\n      retryDelayMs = baseDelayMs * 2 ** attempt;\n    } finally {\n      cleanup();\n    }\n    await sleep(retryDelayMs ?? baseDelayMs * 2 ** attempt, options.signal);\n  }\n  throw new Error('unreachable');\n}\n\n// Any HTTP response means the endpoint is up. Abort it immediately because status does not\n// consume its body, and bound the headers wait for an air-gapped machine.\nexport async function probeReachable(url: string, timeoutMs = 1500): Promise<boolean> {\n  const { controller, cleanup } = attemptController(undefined, timeoutMs);\n  try {\n    const response = await fetch(url, { signal: controller.signal });\n    disposeResponse(response, controller);\n    return true;\n  } catch {\n    return false;\n  } finally {\n    cleanup();\n  }\n}\n"],"names":["HTTP_ATTEMPT_TIMEOUT_MS","MODEL_FILE_ATTEMPT_TIMEOUT_MS","fetchWithRetry","probeReachable","MAX_ATTEMPTS","MAX_RETRY_AFTER_MS","DEFAULT_BASE_DELAY_MS","safeUrl","url","parsed","URL","username","password","search","hash","href","attemptController","callerSignal","timeoutMs","throwIfAborted","controller","AbortController","timeoutReason","relayAbort","abort","reason","addEventListener","once","aborted","timer","setTimeout","cleanup","clearTimeout","removeEventListener","disposeResponse","response","cancellation","body","bodyUsed","cancel","undefined","then","sleep","ms","signal","Promise","resolve","reject","onAbort","failedRequest","phase","attempts","timedOut","endpoint","SenseError","buffered","requestUrl","ok","status","headers","init","options","baseDelayMs","attempt","retryDelayMs","retryable","retryAfter","attemptTimeoutMs","fetch","Number","get","Math","min","ArrayBuffer","arrayBuffer","Error"],"mappings":";;;;;;;;;;;QAOaA;eAAAA;;QACAC;eAAAA;;QA+GSC;eAAAA;;QA+CAC;eAAAA;;;wBAtKK;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAE3B,IAAMC,eAAe;AACrB,IAAMC,qBAAqB;AAC3B,IAAMC,wBAAwB;AAGvB,IAAMN,0BAA0B;AAChC,IAAMC,gCAAgC,IAAI;AAoBjD,SAASM,QAAQC,GAAW;IAC1B,IAAI;QACF,IAAMC,SAAS,IAAIC,IAAIF;QACvBC,OAAOE,QAAQ,GAAG;QAClBF,OAAOG,QAAQ,GAAG;QAClBH,OAAOI,MAAM,GAAG;QAChBJ,OAAOK,IAAI,GAAG;QACd,OAAOL,OAAOM,IAAI;IACpB,EAAE,eAAM;QACN,OAAO;IACT;AACF;AAEA,SAASC,kBACPC,YAAqC,EACrCC,SAAiB;IAMjBD,yBAAAA,mCAAAA,aAAcE,cAAc;IAC5B,IAAMC,aAAa,IAAIC;IACvB,IAAMC,gBAAgB,CAAC;IACvB,IAAMC,aAAa;eAAMH,WAAWI,KAAK,CAACP,yBAAAA,mCAAAA,aAAcQ,MAAM;;IAC9DR,yBAAAA,mCAAAA,aAAcS,gBAAgB,CAAC,SAASH,YAAY;QAAEI,MAAM;IAAK;IACjE,IAAIV,yBAAAA,mCAAAA,aAAcW,OAAO,EAAEL;IAC3B,IAAMM,QAAQC,WAAW;eAAMV,WAAWI,KAAK,CAACF;OAAgBJ;IAChE,OAAO;QACLE,YAAAA;QACAE,eAAAA;QACAS,SAAS,SAATA;YACEC,aAAaH;YACbZ,yBAAAA,mCAAAA,aAAcgB,mBAAmB,CAAC,SAASV;QAC7C;IACF;AACF;AAEA,SAASW,gBAAgBC,QAAkB,EAAEf,UAA2B;IACtE,IAAMgB,eAAeD,SAASE,IAAI,IAAI,CAACF,SAASG,QAAQ,GAAGH,SAASE,IAAI,CAACE,MAAM,KAAKC;IACpFpB,WAAWI,KAAK;IAChB,gGAAgG;IAChG,IAAIY,cACF,KAAKA,aAAaK,IAAI,CACpB;eAAMD;OACN;eAAMA;;AAEZ;AAEA,SAASE,MAAMC,EAAU,EAAEC,MAA+B;IACxDA,mBAAAA,6BAAAA,OAAQzB,cAAc;IACtB,IAAI,CAACyB,QAAQ,OAAO,IAAIC,QAAQ,SAACC;eAAYhB,WAAWgB,SAASH;;IACjE,OAAO,IAAIE,QAAQ,SAACC,SAASC;QAC3B,IAAIlB;QACJ,IAAME,UAAU;YACd,IAAIF,OAAOG,aAAaH;YACxBe,OAAOX,mBAAmB,CAAC,SAASe;QACtC;QACA,IAAMA,UAAU;YACdjB;YACAgB,OAAOH,OAAOnB,MAAM;QACtB;QACAmB,OAAOlB,gBAAgB,CAAC,SAASsB,SAAS;YAAErB,MAAM;QAAK;QACvDE,QAAQC,WAAW;YACjBC;YACAe;QACF,GAAGH;QACH,IAAIC,OAAOhB,OAAO,EAAEoB;IACtB;AACF;AAEA,SAASC,cAAczC,GAAW,EAAE0C,KAAoB,EAAEC,QAAgB,EAAEjC,SAAiB,EAAEkC,QAAiB;IAC9G,IAAMC,WAAW9C,QAAQC;IACzB,IAAI4C,UAAU;QACZ,OAAO,IAAIE,oBAAU,CAAC,eAAe,AAAC,GAA8BpC,OAA5BmC,UAAS,qBAAoDH,OAAjChC,WAAU,yBAAyCiC,OAAlBD,OAAM,cAAwB9C,OAAZ+C,UAAS,KAAgB,OAAb/C,cAAa;IAClJ;IACA,OAAO,IAAIkD,oBAAU,CAAC,eAAe,AAAC,GAAuCJ,OAArCG,UAAS,8BAA2CF,OAAfD,OAAM,WAAkB,OAATC,UAAS;AACvG;AAEA,SAASI,SAASpB,QAAkB,EAAEE,IAAiB,EAAEmB,UAAkB;IACzE,OAAO;QACLC,IAAItB,SAASsB,EAAE;QACfC,QAAQvB,SAASuB,MAAM;QACvBC,SAASxB,SAASwB,OAAO;QACzBtB,MAAAA;QACA7B,KAAKD,QAAQ4B,SAAS3B,GAAG,IAAIgD;IAC/B;AACF;AAIO,SAAetD;wCAAeM,GAAW,EAAEoD,IAAsB;YAAEC,SACtDA,2BACEA,sBADd3C,WACA4C,aACGC,SACPF,iBAC+C7C,oBAAvCI,YAAYE,eAAeS,SAC/BI,UACAe,OACAc,cAIMC,WAEEC,YAKNL,kBAMFA,kBADMxB,cAMNwB,kBAIIT;;;;;oBAlC8DS,UAAAA,oEAA6B,CAAC;oBAChG3C,aAAY2C,4BAAAA,QAAQM,gBAAgB,cAAxBN,uCAAAA,4BAA4B7D;oBACxC8D,eAAcD,uBAAAA,QAAQC,WAAW,cAAnBD,kCAAAA,uBAAuBvD;oBAClCyD,UAAU;;;yBAAGA,CAAAA,UAAU3D,YAAW;;;;qBACzCyD,kBAAAA,QAAQjB,MAAM,cAAdiB,sCAAAA,gBAAgB1C,cAAc;oBACiBH,qBAAAA,kBAAkB6C,QAAQjB,MAAM,EAAE1B,YAAzEE,aAAuCJ,mBAAvCI,YAAYE,gBAA2BN,mBAA3BM,eAAeS,UAAYf,mBAAZe;oBAC/BI,WAAAA,KAAAA;oBACAe,QAAuB;oBACvBc,eAAAA,KAAAA;;;;;;;;;oBAES;;wBAAMI,MAAM5D,KAAK,wCAAKoD;4BAAMhB,QAAQxB,WAAWwB,MAAM;;;;oBAAhET,WAAW;yBACP,CAACA,SAASsB,EAAE,EAAZ;;;;oBACIQ,YAAY9B,SAASuB,MAAM,KAAK,OAAOvB,SAASuB,MAAM,IAAI;oBAChE,IAAIO,aAAaF,UAAU3D,eAAe,GAAG;wBACrC8D,aAAaG,OAAOlC,SAASwB,OAAO,CAACW,GAAG,CAAC;wBAC/CN,eAAeE,aAAa,IAAIK,KAAKC,GAAG,CAACN,aAAa,MAAM7D,sBAAsByD,uBAAc,GAAKC;wBACrG7B,gBAAgBC,UAAUf;oBAC5B,OAAO;;wBACLc,gBAAgBC,UAAUf;yBAC1ByC,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;wBAC9B;;4BAAOoC,SAASpB,UAAU,IAAIsC,YAAY,IAAIjE;;oBAChD;;;;;;oBAEA0C,QAAQ;oBACK;;wBAAMf,SAASuC,WAAW;;;oBAAjCrC,OAAO;qBACbwB,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;oBAC9B;;wBAAOoC,SAASpB,UAAUE,MAAM7B;;;;;;;;;oBAGlC,IAAI;;yBACFqD,mBAAAA,QAAQjB,MAAM,cAAdiB,uCAAAA,iBAAgB1C,cAAc;oBAChC,SAAU;wBACR,IAAIgB,UAAUD,gBAAgBC,UAAUf;oBAC1C;oBACMgC,WAAWhC,WAAWwB,MAAM,CAACnB,MAAM,KAAKH;oBAC9C,IAAIyC,YAAY3D,eAAe,GAAG,MAAM6C,cAAczC,KAAK0C,OAAOa,UAAU,GAAG7C,WAAWkC;oBAC1FY,eAAeF,uBAAc,GAAKC;;;;;;oBAElChC;;;;;oBAEF;;wBAAMW,MAAMsB,yBAAAA,0BAAAA,eAAgBF,uBAAc,GAAKC,UAASF,QAAQjB,MAAM;;;oBAAtE;;;oBArC4CmB;;;;;;oBAuC9C,MAAM,IAAIY,MAAM;;;IAClB;;AAIO,SAAexE,eAAeK,GAAW;QAAEU,YAAAA,iEAAY;;YAC5BF,oBAAxBI,YAAYW,SAEZI;;;;oBAFwBnB,qBAAAA,kBAAkBwB,WAAWtB,YAArDE,aAAwBJ,mBAAxBI,YAAYW,UAAYf,mBAAZe;;;;;;;;;oBAED;;wBAAMqC,MAAM5D,KAAK;4BAAEoC,QAAQxB,WAAWwB,MAAM;wBAAC;;;oBAAxDT,WAAW;oBACjBD,gBAAgBC,UAAUf;oBAC1B;;wBAAO;;;;oBAEP;;wBAAO;;;oBAEPW;;;;;;;;;;IAEJ"}

package/dist/cjs/embed/openai.js

@@ -9,6 +9,7 @@ Object.defineProperty(exports, "openaiProvider", {
     }
 });
 var _errorsts = require("../errors.js");
+var _endpointts = require("./endpoint.js");
 var _httpts = require("./http.js");
 function _array_like_to_array(arr, len) {
     if (len == null || len > arr.length) len = arr.length;
@@ -154,6 +155,10 @@ function _ts_generator(thisArg, body) {
         };
     }
 }
+function _type_of(obj) {
+    "@swc/helpers - typeof";
+    return obj && typeof Symbol !== "undefined" && obj.constructor === Symbol ? "symbol" : typeof obj;
+}
 function _unsupported_iterable_to_array(o, minLen) {
     if (!o) return;
     if (typeof o === "string") return _array_like_to_array(o, minLen);
@@ -163,12 +168,30 @@ function _unsupported_iterable_to_array(o, minLen) {
     if (n === "Arguments" || /^(?:Ui|I)nt(?:8|16|32)(?:Clamped)?Array$/.test(n)) return _array_like_to_array(o, minLen);
 }
 var BATCH_CAP = 64;
+function isEmbedding(value) {
+    if (value === null || (typeof value === "undefined" ? "undefined" : _type_of(value)) !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every(function(part) {
+        return typeof part === 'number';
+    })) return false;
+    return !('index' in value) || value.index === undefined || typeof value.index === 'number';
+}
+function parseBody(body, endpoint) {
+    var decoded;
+    try {
+        decoded = JSON.parse(new TextDecoder().decode(body));
+    } catch (unused) {
+        throw new _errorsts.SenseError('EMBED_MODEL', "".concat(endpoint, " returned invalid JSON"));
+    }
+    if (decoded === null || (typeof decoded === "undefined" ? "undefined" : _type_of(decoded)) !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {
+        throw new _errorsts.SenseError('EMBED_MODEL', "".concat(endpoint, " returned an invalid embedding response"));
+    }
+    return decoded.data;
+}
 function openaiProvider(model, url, keyEnv) {
     return _async_to_generator(function() {
         var base, headers, key, dims;
-        function post(texts) {
+        function post(texts, options) {
             return _async_to_generator(function() {
-                var res, body, data;
+                var _options_signal, res, body, data;
                 return _ts_generator(this, function(_state) {
                     switch(_state.label){
                         case 0:
@@ -181,27 +204,26 @@ function openaiProvider(model, url, keyEnv) {
                                         model: model,
                                         input: texts
                                     })
+                                }, {
+                                    signal: options === null || options === void 0 ? void 0 : options.signal,
+                                    attemptTimeoutMs: _httpts.HTTP_ATTEMPT_TIMEOUT_MS
                                 })
                             ];
                         case 1:
                             res = _state.sent();
-                            if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/embeddings -> HTTP ").concat(res.status));
-                            return [
-                                4,
-                                res.json()
-                            ];
-                        case 2:
-                            body = _state.sent();
+                            options === null || options === void 0 ? void 0 : (_options_signal = options.signal) === null || _options_signal === void 0 ? void 0 : _options_signal.throwIfAborted();
+                            if (!res.ok) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(res.url, " -> HTTP ").concat(res.status));
+                            body = parseBody(res.body, res.url);
                             // The schema carries `index` because response order is not promised, and the caller maps
                             // vectors back to chunks positionally: sort by it whenever every element has one.
-                            data = body.data.every(function(d) {
+                            data = body.every(function(d) {
                                 return typeof d.index === 'number';
-                            }) ? _to_consumable_array(body.data).sort(function(a, b) {
+                            }) ? _to_consumable_array(body).sort(function(a, b) {
                                 return a.index - b.index;
-                            }) : body.data;
+                            }) : body;
                             // A server that drops an input (empty or over-length strings are the usual cause) would
                             // otherwise shift every later vector onto the wrong chunk, silently.
-                            if (data.length !== texts.length) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(base, "/embeddings returned ").concat(data.length, " embeddings for ").concat(texts.length, " inputs"));
+                            if (data.length !== texts.length) throw new _errorsts.SenseError('EMBED_MODEL', "".concat(res.url, " returned ").concat(data.length, " embeddings for ").concat(texts.length, " inputs"));
                             return [
                                 2,
                                 data.map(function(d) {
@@ -215,8 +237,8 @@ function openaiProvider(model, url, keyEnv) {
         return _ts_generator(this, function(_state) {
             switch(_state.label){
                 case 0:
-                    if (!url) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
-                    base = url.replace(/\/+$/, '');
+                    base = (0, _endpointts.effectiveEmbedEndpoint)('openai', url);
+                    if (!base) throw new _errorsts.SenseError('EMBED_MODEL', 'embed.provider "openai" requires a url');
                     headers = {
                         'content-type': 'application/json'
                     };
@@ -239,7 +261,7 @@ function openaiProvider(model, url, keyEnv) {
                             dims: dims,
                             batchCap: BATCH_CAP,
                             embedDocuments: post,
-                            embedQuery: function embedQuery(text) {
+                            embedQuery: function embedQuery(text, options) {
                                 return _async_to_generator(function() {
                                     return _ts_generator(this, function(_state) {
                                         switch(_state.label){
@@ -248,7 +270,7 @@ function openaiProvider(model, url, keyEnv) {
                                                     4,
                                                     post([
                                                         text
-                                                    ])
+                                                    ], options)
                                                 ];
                                             case 1:
                                                 return [

package/dist/cjs/embed/openai.js.map

@@ -1 +1 @@
-{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { fetchWithRetry } from './http.ts';\nimport type { EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  if (!url) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const base = url.replace(/\\/+$/, '');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[]): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) });\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${base}/embeddings -> HTTP ${res.status}`);\n    const body = (await res.json()) as { data: Array<{ embedding: number[]; index?: number }> };\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.data.every((d) => typeof d.index === 'number') ? [...body.data].sort((a, b) => (a.index as number) - (b.index as number)) : body.data;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${base}/embeddings returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text) => (await post([text]))[0] };\n}\n"],"names":["openaiProvider","BATCH_CAP","model","url","keyEnv","base","headers","key","dims","post","texts","res","body","data","fetchWithRetry","method","JSON","stringify","input","ok","SenseError","status","json","every","d","index","sort","a","b","length","map","Float32Array","from","embedding","replace","process","env","undefined","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAQsBA;;;eAAAA;;;wBARK;sBACI;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAG/B,IAAMC,YAAY;AAIX,SAAeD,eAAeE,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAE/FC,MACAC,SACAC,KAgBAC;QAbN,SAAeC,KAAKC,KAAe;;oBAC3BC,KAEAC,MAGAC;;;;4BALM;;gCAAMC,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALT,MAAK,gBAAc;oCAAEU,QAAQ;oCAAQT,SAAAA;oCAASM,MAAMI,KAAKC,SAAS,CAAC;wCAAEf,OAAAA;wCAAOgB,OAAOR;oCAAM;gCAAG;;;4BAA1HC,MAAM;4BACZ,IAAI,CAACA,IAAIQ,EAAE,EAAE,MAAM,IAAIC,oBAAU,CAAC,eAAe,AAAC,GAA6BT,OAA3BN,MAAK,wBAAiC,OAAXM,IAAIU,MAAM;4BAC3E;;gCAAMV,IAAIW,IAAI;;;4BAAtBV,OAAQ;4BACd,yFAAyF;4BACzF,kFAAkF;4BAC5EC,OAAOD,KAAKC,IAAI,CAACU,KAAK,CAAC,SAACC;uCAAM,OAAOA,EAAEC,KAAK,KAAK;iCAAY,AAAC,qBAAGb,KAAKC,IAAI,EAAEa,IAAI,CAAC,SAACC,GAAGC;uCAAM,AAACD,EAAEF,KAAK,GAAeG,EAAEH,KAAK;iCAAeb,KAAKC,IAAI;4BACvJ,wFAAwF;4BACxF,qEAAqE;4BACrE,IAAIA,KAAKgB,MAAM,KAAKnB,MAAMmB,MAAM,EAAE,MAAM,IAAIT,oBAAU,CAAC,eAAe,AAAC,GAA8BP,OAA5BR,MAAK,yBAAqDK,OAA9BG,KAAKgB,MAAM,EAAC,oBAA+B,OAAbnB,MAAMmB,MAAM,EAAC;4BAChJ;;gCAAOhB,KAAKiB,GAAG,CAAC,SAACN;2CAAMO,aAAaC,IAAI,CAACR,EAAES,SAAS;;;;;YACtD;;;;;oBAjBA,IAAI,CAAC9B,KAAK,MAAM,IAAIiB,oBAAU,CAAC,eAAe;oBACxCf,OAAOF,IAAI+B,OAAO,CAAC,QAAQ;oBAC3B5B,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS+B,QAAQC,GAAG,CAAChC,OAAO,GAAGiC;oBAC3C,IAAI9B,KAAKD,QAAQgC,aAAa,GAAG,AAAC,UAAa,OAAJ/B;oBAe7B;;wBAAME;4BAAM;;;;oBAApBD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACqB,MAAM;oBACxD,oFAAoF;oBACpF,gDAAgD;oBAChD;;wBAAO;4BAAEU,IAAI,AAAC,UAAiBrC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAMgC,UAAUvC;4BAAWwC,gBAAgBhC;4BAAMiC,YAAY,SAAZA,WAAmBC;;;;;gDAAU;;oDAAMlC;wDAAMkC;;;;;;oDAAZ,aAAmB,CAAC,EAAE;;;;;;wBAAC;;;;IAC/I"}
+{"version":3,"sources":["/Users/kevin/Dev/OpenSource/ai/sensemaking/src/embed/openai.ts"],"sourcesContent":["import { SenseError } from '../errors.ts';\nimport { effectiveEmbedEndpoint } from './endpoint.ts';\nimport { fetchWithRetry, HTTP_ATTEMPT_TIMEOUT_MS } from './http.ts';\nimport type { EmbedCallOptions, EmbedProvider } from './types.ts';\n\nconst BATCH_CAP = 64;\n\ninterface OpenaiEmbedding {\n  embedding: number[];\n  index?: number;\n}\n\nfunction isEmbedding(value: unknown): value is OpenaiEmbedding {\n  if (value === null || typeof value !== 'object' || !('embedding' in value) || !Array.isArray(value.embedding) || !value.embedding.every((part) => typeof part === 'number')) return false;\n  return !('index' in value) || value.index === undefined || typeof value.index === 'number';\n}\n\nfunction parseBody(body: ArrayBuffer, endpoint: string): OpenaiEmbedding[] {\n  let decoded: unknown;\n  try {\n    decoded = JSON.parse(new TextDecoder().decode(body));\n  } catch {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned invalid JSON`);\n  }\n  if (decoded === null || typeof decoded !== 'object' || !('data' in decoded) || !Array.isArray(decoded.data) || !decoded.data.every(isEmbedding)) {\n    throw new SenseError('EMBED_MODEL', `${endpoint} returned an invalid embedding response`);\n  }\n  return decoded.data;\n}\n\n// One POST against any OpenAI-compatible /embeddings endpoint: Ollama, LM Studio, Cloudflare\n// Workers AI, Jina, Voyage, and Gemini's compat base URL all serve this shape.\nexport async function openaiProvider(model: string, url: string | undefined, keyEnv: string | undefined): Promise<EmbedProvider> {\n  const base = effectiveEmbedEndpoint('openai', url);\n  if (!base) throw new SenseError('EMBED_MODEL', 'embed.provider \"openai\" requires a url');\n  const headers: Record<string, string> = { 'content-type': 'application/json' };\n  const key = keyEnv ? process.env[keyEnv] : undefined;\n  if (key) headers.authorization = `Bearer ${key}`;\n\n  async function post(texts: string[], options?: EmbedCallOptions): Promise<Float32Array[]> {\n    const res = await fetchWithRetry(`${base}/embeddings`, { method: 'POST', headers, body: JSON.stringify({ model, input: texts }) }, { signal: options?.signal, attemptTimeoutMs: HTTP_ATTEMPT_TIMEOUT_MS });\n    options?.signal?.throwIfAborted();\n    if (!res.ok) throw new SenseError('EMBED_MODEL', `${res.url} -> HTTP ${res.status}`);\n    const body = parseBody(res.body, res.url);\n    // The schema carries `index` because response order is not promised, and the caller maps\n    // vectors back to chunks positionally: sort by it whenever every element has one.\n    const data = body.every((d): d is OpenaiEmbedding & { index: number } => typeof d.index === 'number') ? [...body].sort((a, b) => a.index - b.index) : body;\n    // A server that drops an input (empty or over-length strings are the usual cause) would\n    // otherwise shift every later vector onto the wrong chunk, silently.\n    if (data.length !== texts.length) throw new SenseError('EMBED_MODEL', `${res.url} returned ${data.length} embeddings for ${texts.length} inputs`);\n    return data.map((d) => Float32Array.from(d.embedding));\n  }\n\n  const dims = (await post(['dimension probe']))[0].length;\n  // embedQuery and embedDocuments are symmetric here; per-model prefixes like nomic's\n  // search_document/search_query are not applied.\n  return { id: `openai:${base}:${model}`, dims, batchCap: BATCH_CAP, embedDocuments: post, embedQuery: async (text, options) => (await post([text], options))[0] };\n}\n"],"names":["openaiProvider","BATCH_CAP","isEmbedding","value","Array","isArray","embedding","every","part","index","undefined","parseBody","body","endpoint","decoded","JSON","parse","TextDecoder","decode","SenseError","data","model","url","keyEnv","base","headers","key","dims","post","texts","options","res","fetchWithRetry","method","stringify","input","signal","attemptTimeoutMs","HTTP_ATTEMPT_TIMEOUT_MS","throwIfAborted","ok","status","d","sort","a","b","length","map","Float32Array","from","effectiveEmbedEndpoint","process","env","authorization","id","batchCap","embedDocuments","embedQuery","text"],"mappings":";;;;+BAgCsBA;;;eAAAA;;;wBAhCK;0BACY;sBACiB;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;;AAGxD,IAAMC,YAAY;AAOlB,SAASC,YAAYC,KAAc;IACjC,IAAIA,UAAU,QAAQ,CAAA,OAAOA,sCAAP,SAAOA,MAAI,MAAM,YAAY,CAAE,CAAA,eAAeA,KAAI,KAAM,CAACC,MAAMC,OAAO,CAACF,MAAMG,SAAS,KAAK,CAACH,MAAMG,SAAS,CAACC,KAAK,CAAC,SAACC;eAAS,OAAOA,SAAS;QAAW,OAAO;IACpL,OAAO,CAAE,CAAA,WAAWL,KAAI,KAAMA,MAAMM,KAAK,KAAKC,aAAa,OAAOP,MAAMM,KAAK,KAAK;AACpF;AAEA,SAASE,UAAUC,IAAiB,EAAEC,QAAgB;IACpD,IAAIC;IACJ,IAAI;QACFA,UAAUC,KAAKC,KAAK,CAAC,IAAIC,cAAcC,MAAM,CAACN;IAChD,EAAE,eAAM;QACN,MAAM,IAAIO,oBAAU,CAAC,eAAe,AAAC,GAAW,OAATN,UAAS;IAClD;IACA,IAAIC,YAAY,QAAQ,CAAA,OAAOA,wCAAP,SAAOA,QAAM,MAAM,YAAY,CAAE,CAAA,UAAUA,OAAM,KAAM,CAACV,MAAMC,OAAO,CAACS,QAAQM,IAAI,KAAK,CAACN,QAAQM,IAAI,CAACb,KAAK,CAACL,cAAc;QAC/I,MAAM,IAAIiB,oBAAU,CAAC,eAAe,AAAC,GAAW,OAATN,UAAS;IAClD;IACA,OAAOC,QAAQM,IAAI;AACrB;AAIO,SAAepB,eAAeqB,KAAa,EAAEC,GAAuB,EAAEC,MAA0B;;YAC/FC,MAEAC,SACAC,KAiBAC;QAdN,SAAeC,KAAKC,KAAe,EAAEC,OAA0B;;oBAE7DA,iBADMC,KAGAnB,MAGAQ;;;;4BANM;;gCAAMY,IAAAA,sBAAc,EAAC,AAAC,GAAO,OAALR,MAAK,gBAAc;oCAAES,QAAQ;oCAAQR,SAAAA;oCAASb,MAAMG,KAAKmB,SAAS,CAAC;wCAAEb,OAAAA;wCAAOc,OAAON;oCAAM;gCAAG,GAAG;oCAAEO,MAAM,EAAEN,oBAAAA,8BAAAA,QAASM,MAAM;oCAAEC,kBAAkBC,+BAAuB;gCAAC;;;4BAAlMP,MAAM;4BACZD,oBAAAA,+BAAAA,kBAAAA,QAASM,MAAM,cAAfN,sCAAAA,gBAAiBS,cAAc;4BAC/B,IAAI,CAACR,IAAIS,EAAE,EAAE,MAAM,IAAIrB,oBAAU,CAAC,eAAe,AAAC,GAAqBY,OAAnBA,IAAIT,GAAG,EAAC,aAAsB,OAAXS,IAAIU,MAAM;4BAC3E7B,OAAOD,UAAUoB,IAAInB,IAAI,EAAEmB,IAAIT,GAAG;4BACxC,yFAAyF;4BACzF,kFAAkF;4BAC5EF,OAAOR,KAAKL,KAAK,CAAC,SAACmC;uCAAgD,OAAOA,EAAEjC,KAAK,KAAK;iCAAY,AAAC,qBAAGG,MAAM+B,IAAI,CAAC,SAACC,GAAGC;uCAAMD,EAAEnC,KAAK,GAAGoC,EAAEpC,KAAK;iCAAIG;4BACtJ,wFAAwF;4BACxF,qEAAqE;4BACrE,IAAIQ,KAAK0B,MAAM,KAAKjB,MAAMiB,MAAM,EAAE,MAAM,IAAI3B,oBAAU,CAAC,eAAe,AAAC,GAAsBC,OAApBW,IAAIT,GAAG,EAAC,cAA0CO,OAA9BT,KAAK0B,MAAM,EAAC,oBAA+B,OAAbjB,MAAMiB,MAAM,EAAC;4BACxI;;gCAAO1B,KAAK2B,GAAG,CAAC,SAACL;2CAAMM,aAAaC,IAAI,CAACP,EAAEpC,SAAS;;;;;YACtD;;;;;oBAlBMkB,OAAO0B,IAAAA,kCAAsB,EAAC,UAAU5B;oBAC9C,IAAI,CAACE,MAAM,MAAM,IAAIL,oBAAU,CAAC,eAAe;oBACzCM,UAAkC;wBAAE,gBAAgB;oBAAmB;oBACvEC,MAAMH,SAAS4B,QAAQC,GAAG,CAAC7B,OAAO,GAAGb;oBAC3C,IAAIgB,KAAKD,QAAQ4B,aAAa,GAAG,AAAC,UAAa,OAAJ3B;oBAgB7B;;wBAAME;4BAAM;;;;oBAApBD,OAAO,AAAC,aAAgC,CAAC,EAAE,CAACmB,MAAM;oBACxD,oFAAoF;oBACpF,gDAAgD;oBAChD;;wBAAO;4BAAEQ,IAAI,AAAC,UAAiBjC,OAARG,MAAK,KAAS,OAANH;4BAASM,MAAAA;4BAAM4B,UAAUtD;4BAAWuD,gBAAgB5B;4BAAM6B,YAAY,SAAZA,WAAmBC,MAAM5B;;;;;gDAAa;;oDAAMF;wDAAM8B;uDAAO5B;;;;;oDAAnB,aAA4B,CAAC,EAAE;;;;;;wBAAC;;;;IACjK"}

package/dist/cjs/embed/query.d.cts

@@ -1,12 +1,18 @@
-import type { Config } from '../config/index.js';
+import { type Config } from '../config/index.js';
 import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
+import { type EmbedCallOptions } from './types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options?: EmbedCallOptions): Promise<void>;
 export declare function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void>;
-export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export interface PreparedSemanticQuery {
+    readonly vector: Float32Array;
+    readonly storeDims: number;
+}
+export declare function prepareSemanticQuery(cfg: Config, terms: string, options?: EmbedCallOptions): Promise<PreparedSemanticQuery>;
+export declare function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
 export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;

package/dist/cjs/embed/query.d.ts

@@ -1,12 +1,18 @@
-import type { Config } from '../config/index.js';
+import { type Config } from '../config/index.js';
 import type { Store, VectorCandidate, VectorSimilar } from '../store/types.js';
+import { type EmbedCallOptions } from './types.js';
 export declare function toStore(full: Float32Array, dims: number, int8: boolean): {
     v: Float32Array;
     scale: number;
 };
-export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>): Promise<void>;
+export declare function embedPending(store: Store, cfg: Config, allowed?: ReadonlySet<string>, expectedSignature?: string | null, options?: EmbedCallOptions): Promise<void>;
 export declare function ensureDocumentEmbeddings(store: Store, _cfg: Config, allowed: ReadonlySet<string>): Promise<void>;
-export declare function semanticCandidates(store: Store, cfg: Config, terms: string, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
+export interface PreparedSemanticQuery {
+    readonly vector: Float32Array;
+    readonly storeDims: number;
+}
+export declare function prepareSemanticQuery(cfg: Config, terms: string, options?: EmbedCallOptions): Promise<PreparedSemanticQuery>;
+export declare function semanticCandidates(store: Store, prepared: PreparedSemanticQuery, fetch: number, allowed?: Set<string>): Promise<VectorCandidate[]>;
 export declare function hasEmbedding(store: Store, path: string): Promise<boolean>;
 export declare function similarNotes(store: Store, _cfg: Config, path: string, opts: {
     exclude: Set<string>;