pi-llama-cpp 0.12.0 → 0.14.0

package/src/managers/settings.ts

@@ -9,23 +9,22 @@ import { join } from "node:path";
 import {
   API_KEY_PLACEHOLDER,
   AUTOLOAD_ON_MESSAGE,
-  LLAMA_SERVER_URL,
   POLLING_TIMEOUT,
   REACT_TO_MODEL_SELECT,
   SERVER_TIMEOUT,
   SETTINGS_KEY,
   SORT_BY,
   THINKING_BUDGETS,
-  type SortBy,
 } from "../constants";
 import {
   LlamaServer,
   LlamaSettings,
   ModelOverride,
 } from "../interfaces/settings";
+import type { SortBy } from "../interfaces/sortBy";
 import { Server } from "../server";
 import { SettingsStore } from "../utils/settingsStore";
-import { isValidServerUrl, normalizeUrl } from "../utils/urls";
+import { UrlResolver } from "../utils/urlResolver";
 
 export class LlamaSettingsManager {
   private settingsManager = SettingsManager.create(process.cwd());
@@ -47,16 +46,17 @@ export class LlamaSettingsManager {
     }
   }
 
-  /** Warnings collected during URL resolution (dropped invalid entries). */
-  private warnings: string[] = [];
+  /** Delegated multi-source URL resolution chain (see `utils/urlResolver`). */
+  private urlResolver = new UrlResolver({
+    getLlamaSettings: () => this.getLlamaSettings(),
+    getMergedSettings: () => this.getMergedSettings(),
+  });
 
   /**
    * Returns and clears warnings collected during URL resolution.
    */
   takeWarnings(): string[] {
-    const warnings = [...this.warnings];
-    this.warnings.length = 0;
-    return warnings;
+    return this.urlResolver.takeWarnings();
   }
 
   /**
@@ -75,7 +75,7 @@ export class LlamaSettingsManager {
    * Convenience method for the `llamaSettings` key.
    * Reloads settings from disk before reading.
    */
-  async getLlamaSettings(): Promise<LlamaSettings> {
+  private async getLlamaSettings(): Promise<LlamaSettings> {
     return (await this.getMergedSettings())[SETTINGS_KEY] ?? {};
   }
 
@@ -88,89 +88,13 @@ export class LlamaSettingsManager {
   }
 
   /**
-   * Resolves the server URLs to use in the following order:
-   *
-   * - `LLAMA_SERVER_URL` env variable
-   * - `llamaSettings` key (current - project, then global)
-   * - `llamaServerUrl` key (legacy - project, then global)
-   * - Default URL
+   * Resolves the server URLs to use. Delegates to the URL resolver chain
+   * (env → settings → legacy → default, see `utils/urlResolver`).
    *
    * @returns The list of URLs to use
    */
   async resolveUrls(): Promise<string[]> {
-    let response = this.resolveEnvUrls();
-    if (response.length > 0) return response;
-
-    response = await this.resolveServerUrls();
-    if (response.length > 0) return response;
-
-    response = await this.resolveLegacyUrls();
-    if (response.length > 0) return response;
-
-    return [LLAMA_SERVER_URL];
-  }
-
-  /**
-   * Resolves the llama-server URLs from the environment variable.
-   *
-   * @returns A list of detected URLs
-   */
-  private resolveEnvUrls(): string[] {
-    const raw = process.env.LLAMA_SERVER_URL;
-    if (!raw) return [];
-
-    return this.parseUrls(raw);
-  }
-
-  /**
-   * Resolves the llama-server URLs from `llamaSettings.servers`.
-   * Settings are merged, prioritizing project over global settings.
-   * Reloads settings from disk before reading.
-   *
-   * @returns A list of detected URLs
-   */
-  private async resolveServerUrls(): Promise<string[]> {
-    const { servers = [] } = await this.getLlamaSettings();
-    return servers.map((s) => this.parseUrls(s.url)).flat();
-  }
-
-  /**
-   * Resolves the llama-server URLs from `llamaServerUrl` legacy key.
-   * Settings are merged, prioritizing project over global settings.
-   * Reloads settings from disk before reading.
-   *
-   * @returns A list of detected URLs
-   */
-  private async resolveLegacyUrls(): Promise<string[]> {
-    const { llamaServerUrl = null } = await this.getMergedSettings();
-    if (!llamaServerUrl) return [];
-
-    return this.parseUrls(llamaServerUrl);
-  }
-
-  /**
-   * Parses a raw URL string into an array of cleaned URLs.
-   * Splits on semicolons, trims whitespace, filters empty strings, strips
-   * trailing slashes, and drops entries without an http(s) scheme —
-   * collecting a warning for each dropped entry (same validation the
-   * `/models servers` editor applies).
-   *
-   * @returns A sanitized URL
-   */
-  private parseUrls(raw: string): string[] {
-    return raw
-      .split(";")
-      .map(normalizeUrl)
-      .filter((u) => {
-        if (u.length === 0) return false;
-        if (!isValidServerUrl(u)) {
-          this.warnings.push(
-            `Ignoring invalid server URL '${u}' (needs http(s)://)`,
-          );
-          return false;
-        }
-        return true;
-      });
+    return this.urlResolver.resolveUrls();
   }
 
   /**

package/src/models/baseModel.ts

@@ -23,7 +23,6 @@ export abstract class BaseModel {
     [Status.FAILED]: "🔴",
     [Status.SLEEPING]: "🔵",
     [Status.UNLOADED]: "⚪",
-    [Status.UNAUTHORIZED]: "⛔",
   };
 
   abstract get mode(): Mode;
@@ -72,7 +71,7 @@ export abstract class BaseModel {
    *
    * @returns An array of capabilities, as expected by Pi
    */
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     const overridden = this.server.findOverrideForModel(this.id)?.capabilities;
     if (overridden) return overridden;
 
@@ -108,7 +107,6 @@ export abstract class BaseModel {
 
       if (is_sleeping) return Status.SLEEPING;
       if (!error) return Status.LOADED;
-      if (error.code === 401) return Status.UNAUTHORIZED;
       if (error.code === 503) return Status.LOADING;
       if (error.code === 400 && error.message === "model is not loaded")
         return Status.UNLOADED;
@@ -128,14 +126,13 @@ export abstract class BaseModel {
    *
    * @returns The context size in tokens
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     const overridden = this.server.findOverrideForModel(this.id)?.contextSize;
     if (overridden && overridden > 0) return overridden;
 
     try {
       const { data } = await this.server.fetchModels();
-      const { n_ctx } = data.find((m) => m.id === this.id)?.meta!;
-
+      const n_ctx = data.find((m) => m.id === this.id)?.meta?.n_ctx;
       return n_ctx ?? FALLBACK_CTX;
     } catch {
       return FALLBACK_CTX;
@@ -187,6 +184,10 @@ export abstract class BaseModel {
       cacheWrite: userCost.cacheWrite ?? 0,
     };
 
+    const input = await this.getCapabilities();
+    const contextWindow = await this.getContextSize();
+    const maxTokens = this.getMaxTokens(contextWindow);
+
     const response: ProviderModelConfig = {
       id: this.id,
       name: this.name,
@@ -199,10 +200,10 @@ export abstract class BaseModel {
         xhigh: "xhigh",
         max: "max",
       },
-      input: await this.getCapabilities(),
-      contextWindow: await this.getContextSize(),
+      input,
+      contextWindow,
       cost,
-      maxTokens: override.maxTokens ?? (await this.getContextSize()),
+      maxTokens,
     };
 
     // Add compat if the override specifies it
@@ -213,6 +214,22 @@ export abstract class BaseModel {
     return response;
   }
 
+  /**
+   * Gets the maximum number of tokens the model can generate.
+   *
+   * An override's `maxTokens` (when set and `> 0`) replaces the value;
+   * otherwise it falls back to the context size — a model cannot generate
+   * more tokens than its context holds. A stored `0` behaves as if the
+   * key were absent.
+   *
+   * @param contextSize - The already-resolved context size, used as fallback
+   * @returns The maximum number of tokens
+   */
+  protected getMaxTokens(contextSize: number): number {
+    const overridden = this.server.findOverrideForModel(this.id)?.maxTokens;
+    return overridden && overridden > 0 ? overridden : contextSize;
+  }
+
   /**
    * Loads the model in llama-server.
    * Uses SSE status events when available, falling back to polling.
@@ -269,7 +286,7 @@ export abstract class BaseModel {
    * @param timeout The maximum amount of ms before timeout. Defaults to server's pollingTimeout
    * @param interval The polling interval. Defaults to POLLING_INTERVAL
    */
-  async pollStatus(
+  protected async pollStatus(
     startTime: number = Date.now(),
     timeout?: number,
     interval: number = POLLING_INTERVAL,

package/src/models/legacyModel.ts

@@ -13,7 +13,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns The context size
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     const props = await this.server.fetchModelProps(this.id);
     const models = await this.server.fetchModels();
 
@@ -34,7 +34,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns An array of capabilities, as expected by Pi
    */
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/models/routerModel.ts

@@ -26,7 +26,7 @@ export class RouterModel extends BaseModel {
    *
    * In exchange, it will allow unloaded models to be correctly shown as "unloaded".
    */
-  async pollStatus(startTime = Date.now()): Promise<void> {
+  protected async pollStatus(startTime = Date.now()): Promise<void> {
     let elapsed = 0;
     const limit = 5000;
 
@@ -52,7 +52,7 @@ export class RouterModel extends BaseModel {
    *
    * @returns The context size in tokens
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     // We can get a more accurate context size if the model is already loaded
     if ((await this.getStatus()) === Status.LOADED) {
       return super.getContextSize();

package/src/models/singleModel.ts

@@ -6,7 +6,7 @@ export class SingleModel extends BaseModel {
     return Mode.SINGLE;
   }
 
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/server.ts

@@ -1,12 +1,11 @@
 import { ApiClient } from "./api/client";
 import {
   API_KEY_PLACEHOLDER,
+  ENDPOINT_PREFIX,
   PROVIDER_NAME,
-  PROVIDER_PREFIX,
 } from "./constants";
 import { Mode } from "./enums/mode";
 import { ServerStatus } from "./enums/serverStatus";
-import { HealthEndpoint } from "./interfaces/endpoints/health";
 import { ModelsEndpoint } from "./interfaces/endpoints/models";
 import {
   PropsEndpoint,
@@ -20,20 +19,23 @@ import { LegacyModel } from "./models/legacyModel";
 import { RouterModel } from "./models/routerModel";
 import { SingleModel } from "./models/singleModel";
 import { SSEManager } from "./sse/manager";
+import { checkServerHealth } from "./utils/health";
+import { ServerIds } from "./utils/serverIds";
 
 /**
  * Optional constructor collaborators for {@link Server} — the seam tests use
  * to run the real Server against fake clients.
  *
  * Both are factories because their arguments only exist around construction:
- * the API key is (re-)resolved by the Server, and SSEManager needs its owner.
- * Factories must stay pure functions of their arguments — `initialize()`
- * re-invokes both on every scan (the ApiClient rebuild picks up a fresh key,
- * by design), so captured per-server state would leak across re-scans.
+ * the API key is (re-)resolved by the Server, and SSEManager needs its owner
+ * (it reads the key and timeouts live through it). Factories must stay pure
+ * functions of their arguments — `initialize()` re-invokes both on every scan
+ * (the ApiClient rebuild picks up a fresh key, by design), so captured
+ * per-server state would leak across re-scans.
  */
 export type ServerDeps = {
   createApiClient?: (apiKey: string) => ApiClient;
-  createSSEManager?: (server: Server, apiKey: string) => SSEManager;
+  createSSEManager?: (server: Server) => SSEManager;
 };
 
 export class Server {
@@ -59,6 +61,19 @@ export class Server {
     return this.options.baseUrl;
   }
 
+  /**
+   * Base URL of the OpenAI-compatible API: {@link baseUrl} joined with
+   * {@link ENDPOINT_PREFIX}. Idempotent — a baseUrl that already ends with
+   * the prefix is returned untouched, so proxied setups exposing the API
+   * under a /v1 path don't double up.
+   */
+  get apiBaseUrl(): string {
+    const { baseUrl } = this.options;
+    return baseUrl.endsWith(ENDPOINT_PREFIX)
+      ? baseUrl
+      : `${baseUrl}${ENDPOINT_PREFIX}`;
+  }
+
   /**
    * Maximum time (ms) for server verification and SSE support probe.
    * Resolved live from the injected settings manager.
@@ -87,7 +102,7 @@ export class Server {
    * Uses custom ID if provided, otherwise falls back to URL-based ID.
    */
   get providerId(): string {
-    return this.options.customId ?? `${PROVIDER_PREFIX}=${this.baseUrl}`;
+    return ServerIds.resolve(this.baseUrl, this.options.customId);
   }
 
   /**
@@ -114,7 +129,7 @@ export class Server {
       if (key !== API_KEY_PLACEHOLDER) return key;
     }
     // Fall back to URL-based ID
-    return this.settings.resolveApiKey(`${PROVIDER_PREFIX}=${this.baseUrl}`);
+    return this.settings.resolveApiKey(ServerIds.fromUrl(this.baseUrl));
   }
 
   /**
@@ -126,9 +141,7 @@ export class Server {
     this.apiClient =
       this.deps.createApiClient?.(apiKey) ??
       new ApiClient(this.baseUrl, apiKey);
-    this.sse =
-      this.deps.createSSEManager?.(this, apiKey) ??
-      new SSEManager(this, apiKey);
+    this.sse = this.deps.createSSEManager?.(this) ?? new SSEManager(this);
     const { data } = await this.fetchModels();
     const mode = await this.detectServerMode(data);
 
@@ -163,37 +176,16 @@ export class Server {
   /**
    * Checks if the server is ready, with a timeout.
    *
+   * Delegates to the shared health probe (`utils/health`) — bypasses the
+   * `ApiClient` on purpose: each caller probes once per scan, so the
+   * client's cache/dedup would be dead weight, and a plain `fetch` with
+   * `AbortSignal.timeout` actually cancels the request on timeout.
+   *
    * @param timeout Maximum time to wait for the health check
    * @returns The server status
    */
   async isReady(timeout: number): Promise<ServerStatus> {
-    try {
-      const timeoutPromise = new Promise<never>((_, reject) =>
-        setTimeout(() => reject(new Error("timeout")), timeout),
-      );
-      const health = await Promise.race([
-        this.fetchServerHealth(),
-        timeoutPromise,
-      ]);
-      if (health.status === "ok") {
-        return ServerStatus.READY;
-      }
-      return ServerStatus.UNREACHABLE;
-    } catch (error) {
-      if (error instanceof Error && error.message === "timeout") {
-        return ServerStatus.TIMEOUT;
-      }
-      return ServerStatus.UNREACHABLE;
-    }
-  }
-
-  /**
-   * Retrieves the health status of the server
-   *
-   * @returns The health status
-   */
-  async fetchServerHealth(): Promise<HealthEndpoint> {
-    return await this.apiClient.get<HealthEndpoint>("/health");
+    return checkServerHealth(this.baseUrl, timeout, this.getApiKey());
   }
 
   /**
@@ -202,7 +194,9 @@ export class Server {
    * @return The models from the server
    */
   async fetchModels(): Promise<ModelsEndpoint> {
-    return await this.apiClient.get<ModelsEndpoint>("/v1/models");
+    return await this.apiClient.get<ModelsEndpoint>(
+      `${ENDPOINT_PREFIX}/models`,
+    );
   }
 
   /**
@@ -226,13 +220,6 @@ export class Server {
     );
   }
 
-  /**
-   * Returns the per-model override configuration for this server.
-   */
-  getOverrides(): Record<string, ModelOverride> {
-    return this.options.overrides ?? {};
-  }
-
   /**
    * Resolves the override for a given model ID using prefix matching.
    *
@@ -244,7 +231,7 @@ export class Server {
    * @returns The matching override, or `undefined` if no key matches.
    */
   findOverrideForModel(modelId: string): ModelOverride | undefined {
-    const overrides = this.getOverrides();
+    const overrides = this.options.overrides ?? {};
     let best: ModelOverride | undefined;
     let bestLen = 0;
 

package/src/sse/client.ts

@@ -1,30 +1,22 @@
 import { POLLING_INTERVAL } from "../constants";
+import { openSSEStream } from "./fetch";
 import type { SSECallback, SSECleanup, SSEEvent } from "./types";
 
-/**
- * Builds the full SSE endpoint URL, appending the API key as a query
- * parameter when one is set. Shared by {@link SSEClient} and
- * {@link SSEManager.probeSSE} so the two can't drift.
- */
-export const buildSSEUrl = (endpoint: string, apiKey?: string): string => {
-  if (apiKey) {
-    return `${endpoint}?api_key=${encodeURIComponent(apiKey)}`;
-  }
-  return endpoint;
-};
-
 /**
  * SSE client for llama-server's /models/sse endpoint.
  *
- * Uses a single shared EventSource per server instance.
+ * Uses a single shared stream per server instance, consumed with `fetch`
+ * (EventSource cannot send the API key as a header, and llama-server no
+ * longer accepts it as a query parameter — see `fetch.ts`).
  * Supports multiple model subscriptions with automatic event routing.
  * Handles reconnection by re-subscribing all callbacks.
  */
 export class SSEClient {
-  private eventSource: EventSource | null = null;
+  private abortController: AbortController | null = null;
+  private disposed: boolean = false;
   private subscribers: Map<string, SSECallback> = new Map();
   private connected: boolean = false;
-  private reconnecting: boolean = false; // tracks if EventSource auto-reconnect is in progress
+  private reconnecting: boolean = false; // tracks if reconnect is in progress
   /**
    * Single shared slot — each setOnConnectFailed call overwrites the
    * previous callback (see there for the constraint this imposes).
@@ -42,7 +34,15 @@ export class SSEClient {
   ) {}
 
   /**
-   * Connects to the SSE endpoint.
+   * Waits the given amount of time (used between reconnection attempts).
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise<void>((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Connects to the SSE endpoint and keeps it open, reconnecting with a
+   * fixed delay until `disconnect()` is called.
    *
    * No current caller consumes the result: `subscribe()` triggers the
    * connection without awaiting it, and connection failures before the
@@ -50,56 +50,110 @@ export class SSEClient {
    *
    * @returns true if the connection was established successfully
    */
-  async connect(): Promise<boolean> {
+  private async connect(): Promise<boolean> {
     if (this.connected) return true;
+    this.disposed = false;
 
-    const url = buildSSEUrl(this.sseEndpoint, this.apiKey);
+    // Loop-local so a stale, already-aborted controller from a previous
+    // connection can't be confused with the current one after a revival.
+    let abortController: AbortController | null = this.abortController;
 
-    try {
-      this.eventSource = new EventSource(url);
-    } catch {
-      this.connected = false;
-      return false;
-    }
+    while (!this.disposed) {
+      if (abortController?.signal.aborted) return false;
+      abortController = new AbortController();
+      this.abortController = abortController;
 
-    this.eventSource.onerror = () => {
-      // EventSource will auto-reconnect; we just track state
-      this.connected = false;
+      let body: ReadableStream<Uint8Array>;
+      try {
+        body = await openSSEStream(
+          this.sseEndpoint,
+          this.apiKey,
+          abortController.signal,
+        );
+      } catch {
+        if (this.disposed || abortController.signal.aborted) return false;
+        this.notifyConnectFailed();
+        this.reconnecting = true;
+        await this.delay(POLLING_INTERVAL);
+        continue;
+      }
+
+      this.connected = true;
+      this.reconnecting = false;
+
+      await this.consume(body);
+
+      if (this.disposed || abortController.signal.aborted) return false;
+      // Stream ended (server closed or network error): reconnect
       this.reconnecting = true;
+      await this.delay(POLLING_INTERVAL);
+    }
 
-      // Notify subscriber if connection fails before any event is received
-      if (!this._hasReceivedEvents && this._onConnectFailed) {
-        this._onConnectFailed();
-      }
-    };
+    return false;
+  }
 
-    this.eventSource.onmessage = (event: MessageEvent) => {
-      try {
-        const data = JSON.parse(event.data);
-        const sseEvent: SSEEvent = {
-          event: data.event ?? "unknown",
-          model: data.model ?? "*",
-          data: data.data,
-        };
-        this._hasReceivedEvents = true;
-        this.dispatch(sseEvent);
-      } catch {
-        // Invalid JSON, ignore
+  /**
+   * Reads the raw byte stream, parses the SSE framing and dispatches
+   * `data:` payloads. Resolves when the stream ends or errors.
+   */
+  private async consume(body: ReadableStream<Uint8Array>): Promise<void> {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        let newlineIndex: number;
+        while ((newlineIndex = buffer.indexOf("\n")) !== -1) {
+          const line = buffer.slice(0, newlineIndex);
+          buffer = buffer.slice(newlineIndex + 1);
+          this.handleLine(line);
+        }
       }
-    };
+      // Flush a trailing data line that lacked its event-terminating newline
+      if (buffer) this.handleLine(buffer);
+    } catch {
+      // Stream aborted or network error — handled by the reconnect logic
+    } finally {
+      reader.releaseLock();
+    }
+  }
 
-    // Wait a bit for the connection to establish
-    await new Promise<void>((resolve) => {
-      const timeout = setTimeout(() => resolve(), POLLING_INTERVAL);
-      this.eventSource!.onopen = () => {
-        clearTimeout(timeout);
-        this.connected = true;
-        this.reconnecting = false;
-        resolve();
+  /**
+   * Handles a single SSE line, dispatching `data:` payloads as events.
+   */
+  private handleLine(line: string): void {
+    if (!line.startsWith("data:")) return;
+
+    const payload = line.slice(5).trim();
+    if (!payload || payload === "[DONE]") return;
+
+    try {
+      const data = JSON.parse(payload);
+      const sseEvent: SSEEvent = {
+        event: data.event ?? "unknown",
+        model: data.model ?? "*",
+        data: data.data,
       };
-    });
+      this._hasReceivedEvents = true;
+      this.dispatch(sseEvent);
+    } catch {
+      // Invalid JSON, ignore
+    }
+  }
 
-    return this.connected;
+  /**
+   * Notifies the connect-failed callback if the connection failed before
+   * any event was received.
+   */
+  private notifyConnectFailed(): void {
+    if (!this._hasReceivedEvents && this._onConnectFailed) {
+      this._onConnectFailed();
+    }
   }
 
   /**
@@ -143,11 +197,11 @@ export class SSEClient {
    * Disconnects from the SSE endpoint and clears all subscriptions.
    */
   disconnect(): void {
-    if (this.eventSource) {
-      this.eventSource.close();
-      this.eventSource = null;
-    }
+    this.disposed = true;
+    this.abortController?.abort();
+    this.abortController = null;
     this.connected = false;
+    this.reconnecting = false;
     this.subscribers.clear();
   }