pi-llama-cpp 0.13.0 → 0.14.0

package/src/managers/settings.ts

@@ -9,23 +9,22 @@ import { join } from "node:path";
 import {
   API_KEY_PLACEHOLDER,
   AUTOLOAD_ON_MESSAGE,
-  LLAMA_SERVER_URL,
   POLLING_TIMEOUT,
   REACT_TO_MODEL_SELECT,
   SERVER_TIMEOUT,
   SETTINGS_KEY,
   SORT_BY,
   THINKING_BUDGETS,
-  type SortBy,
 } from "../constants";
 import {
   LlamaServer,
   LlamaSettings,
   ModelOverride,
 } from "../interfaces/settings";
+import type { SortBy } from "../interfaces/sortBy";
 import { Server } from "../server";
 import { SettingsStore } from "../utils/settingsStore";
-import { isValidServerUrl, normalizeUrl } from "../utils/urls";
+import { UrlResolver } from "../utils/urlResolver";
 
 export class LlamaSettingsManager {
   private settingsManager = SettingsManager.create(process.cwd());
@@ -47,16 +46,17 @@ export class LlamaSettingsManager {
     }
   }
 
-  /** Warnings collected during URL resolution (dropped invalid entries). */
-  private warnings: string[] = [];
+  /** Delegated multi-source URL resolution chain (see `utils/urlResolver`). */
+  private urlResolver = new UrlResolver({
+    getLlamaSettings: () => this.getLlamaSettings(),
+    getMergedSettings: () => this.getMergedSettings(),
+  });
 
   /**
    * Returns and clears warnings collected during URL resolution.
    */
   takeWarnings(): string[] {
-    const warnings = [...this.warnings];
-    this.warnings.length = 0;
-    return warnings;
+    return this.urlResolver.takeWarnings();
   }
 
   /**
@@ -75,7 +75,7 @@ export class LlamaSettingsManager {
    * Convenience method for the `llamaSettings` key.
    * Reloads settings from disk before reading.
    */
-  async getLlamaSettings(): Promise<LlamaSettings> {
+  private async getLlamaSettings(): Promise<LlamaSettings> {
     return (await this.getMergedSettings())[SETTINGS_KEY] ?? {};
   }
 
@@ -88,89 +88,13 @@ export class LlamaSettingsManager {
   }
 
   /**
-   * Resolves the server URLs to use in the following order:
-   *
-   * - `LLAMA_SERVER_URL` env variable
-   * - `llamaSettings` key (current - project, then global)
-   * - `llamaServerUrl` key (legacy - project, then global)
-   * - Default URL
+   * Resolves the server URLs to use. Delegates to the URL resolver chain
+   * (env → settings → legacy → default, see `utils/urlResolver`).
    *
    * @returns The list of URLs to use
    */
   async resolveUrls(): Promise<string[]> {
-    let response = this.resolveEnvUrls();
-    if (response.length > 0) return response;
-
-    response = await this.resolveServerUrls();
-    if (response.length > 0) return response;
-
-    response = await this.resolveLegacyUrls();
-    if (response.length > 0) return response;
-
-    return [LLAMA_SERVER_URL];
-  }
-
-  /**
-   * Resolves the llama-server URLs from the environment variable.
-   *
-   * @returns A list of detected URLs
-   */
-  private resolveEnvUrls(): string[] {
-    const raw = process.env.LLAMA_SERVER_URL;
-    if (!raw) return [];
-
-    return this.parseUrls(raw);
-  }
-
-  /**
-   * Resolves the llama-server URLs from `llamaSettings.servers`.
-   * Settings are merged, prioritizing project over global settings.
-   * Reloads settings from disk before reading.
-   *
-   * @returns A list of detected URLs
-   */
-  private async resolveServerUrls(): Promise<string[]> {
-    const { servers = [] } = await this.getLlamaSettings();
-    return servers.map((s) => this.parseUrls(s.url)).flat();
-  }
-
-  /**
-   * Resolves the llama-server URLs from `llamaServerUrl` legacy key.
-   * Settings are merged, prioritizing project over global settings.
-   * Reloads settings from disk before reading.
-   *
-   * @returns A list of detected URLs
-   */
-  private async resolveLegacyUrls(): Promise<string[]> {
-    const { llamaServerUrl = null } = await this.getMergedSettings();
-    if (!llamaServerUrl) return [];
-
-    return this.parseUrls(llamaServerUrl);
-  }
-
-  /**
-   * Parses a raw URL string into an array of cleaned URLs.
-   * Splits on semicolons, trims whitespace, filters empty strings, strips
-   * trailing slashes, and drops entries without an http(s) scheme —
-   * collecting a warning for each dropped entry (same validation the
-   * `/models servers` editor applies).
-   *
-   * @returns A sanitized URL
-   */
-  private parseUrls(raw: string): string[] {
-    return raw
-      .split(";")
-      .map(normalizeUrl)
-      .filter((u) => {
-        if (u.length === 0) return false;
-        if (!isValidServerUrl(u)) {
-          this.warnings.push(
-            `Ignoring invalid server URL '${u}' (needs http(s)://)`,
-          );
-          return false;
-        }
-        return true;
-      });
+    return this.urlResolver.resolveUrls();
   }
 
   /**

package/src/models/baseModel.ts

@@ -23,7 +23,6 @@ export abstract class BaseModel {
     [Status.FAILED]: "🔴",
     [Status.SLEEPING]: "🔵",
     [Status.UNLOADED]: "⚪",
-    [Status.UNAUTHORIZED]: "⛔",
   };
 
   abstract get mode(): Mode;
@@ -72,7 +71,7 @@ export abstract class BaseModel {
    *
    * @returns An array of capabilities, as expected by Pi
    */
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     const overridden = this.server.findOverrideForModel(this.id)?.capabilities;
     if (overridden) return overridden;
 
@@ -108,7 +107,6 @@ export abstract class BaseModel {
 
       if (is_sleeping) return Status.SLEEPING;
       if (!error) return Status.LOADED;
-      if (error.code === 401) return Status.UNAUTHORIZED;
       if (error.code === 503) return Status.LOADING;
       if (error.code === 400 && error.message === "model is not loaded")
         return Status.UNLOADED;
@@ -128,14 +126,13 @@ export abstract class BaseModel {
    *
    * @returns The context size in tokens
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     const overridden = this.server.findOverrideForModel(this.id)?.contextSize;
     if (overridden && overridden > 0) return overridden;
 
     try {
       const { data } = await this.server.fetchModels();
-      const { n_ctx } = data.find((m) => m.id === this.id)?.meta!;
-
+      const n_ctx = data.find((m) => m.id === this.id)?.meta?.n_ctx;
       return n_ctx ?? FALLBACK_CTX;
     } catch {
       return FALLBACK_CTX;
@@ -187,6 +184,10 @@ export abstract class BaseModel {
       cacheWrite: userCost.cacheWrite ?? 0,
     };
 
+    const input = await this.getCapabilities();
+    const contextWindow = await this.getContextSize();
+    const maxTokens = this.getMaxTokens(contextWindow);
+
     const response: ProviderModelConfig = {
       id: this.id,
       name: this.name,
@@ -199,10 +200,10 @@ export abstract class BaseModel {
         xhigh: "xhigh",
         max: "max",
       },
-      input: await this.getCapabilities(),
-      contextWindow: await this.getContextSize(),
+      input,
+      contextWindow,
       cost,
-      maxTokens: override.maxTokens ?? (await this.getContextSize()),
+      maxTokens,
     };
 
     // Add compat if the override specifies it
@@ -213,6 +214,22 @@ export abstract class BaseModel {
     return response;
   }
 
+  /**
+   * Gets the maximum number of tokens the model can generate.
+   *
+   * An override's `maxTokens` (when set and `> 0`) replaces the value;
+   * otherwise it falls back to the context size — a model cannot generate
+   * more tokens than its context holds. A stored `0` behaves as if the
+   * key were absent.
+   *
+   * @param contextSize - The already-resolved context size, used as fallback
+   * @returns The maximum number of tokens
+   */
+  protected getMaxTokens(contextSize: number): number {
+    const overridden = this.server.findOverrideForModel(this.id)?.maxTokens;
+    return overridden && overridden > 0 ? overridden : contextSize;
+  }
+
   /**
    * Loads the model in llama-server.
    * Uses SSE status events when available, falling back to polling.
@@ -269,7 +286,7 @@ export abstract class BaseModel {
    * @param timeout The maximum amount of ms before timeout. Defaults to server's pollingTimeout
    * @param interval The polling interval. Defaults to POLLING_INTERVAL
    */
-  async pollStatus(
+  protected async pollStatus(
     startTime: number = Date.now(),
     timeout?: number,
     interval: number = POLLING_INTERVAL,

package/src/models/legacyModel.ts

@@ -13,7 +13,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns The context size
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     const props = await this.server.fetchModelProps(this.id);
     const models = await this.server.fetchModels();
 
@@ -34,7 +34,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns An array of capabilities, as expected by Pi
    */
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/models/routerModel.ts

@@ -26,7 +26,7 @@ export class RouterModel extends BaseModel {
    *
    * In exchange, it will allow unloaded models to be correctly shown as "unloaded".
    */
-  async pollStatus(startTime = Date.now()): Promise<void> {
+  protected async pollStatus(startTime = Date.now()): Promise<void> {
     let elapsed = 0;
     const limit = 5000;
 
@@ -52,7 +52,7 @@ export class RouterModel extends BaseModel {
    *
    * @returns The context size in tokens
    */
-  async getContextSize(): Promise<number> {
+  protected async getContextSize(): Promise<number> {
     // We can get a more accurate context size if the model is already loaded
     if ((await this.getStatus()) === Status.LOADED) {
       return super.getContextSize();

package/src/models/singleModel.ts

@@ -6,7 +6,7 @@ export class SingleModel extends BaseModel {
     return Mode.SINGLE;
   }
 
-  async getCapabilities(): Promise<("text" | "image")[]> {
+  protected async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/server.ts

@@ -3,11 +3,9 @@ import {
   API_KEY_PLACEHOLDER,
   ENDPOINT_PREFIX,
   PROVIDER_NAME,
-  PROVIDER_PREFIX,
 } from "./constants";
 import { Mode } from "./enums/mode";
 import { ServerStatus } from "./enums/serverStatus";
-import { HealthEndpoint } from "./interfaces/endpoints/health";
 import { ModelsEndpoint } from "./interfaces/endpoints/models";
 import {
   PropsEndpoint,
@@ -22,20 +20,22 @@ import { RouterModel } from "./models/routerModel";
 import { SingleModel } from "./models/singleModel";
 import { SSEManager } from "./sse/manager";
 import { checkServerHealth } from "./utils/health";
+import { ServerIds } from "./utils/serverIds";
 
 /**
  * Optional constructor collaborators for {@link Server} — the seam tests use
  * to run the real Server against fake clients.
  *
  * Both are factories because their arguments only exist around construction:
- * the API key is (re-)resolved by the Server, and SSEManager needs its owner.
- * Factories must stay pure functions of their arguments — `initialize()`
- * re-invokes both on every scan (the ApiClient rebuild picks up a fresh key,
- * by design), so captured per-server state would leak across re-scans.
+ * the API key is (re-)resolved by the Server, and SSEManager needs its owner
+ * (it reads the key and timeouts live through it). Factories must stay pure
+ * functions of their arguments — `initialize()` re-invokes both on every scan
+ * (the ApiClient rebuild picks up a fresh key, by design), so captured
+ * per-server state would leak across re-scans.
  */
 export type ServerDeps = {
   createApiClient?: (apiKey: string) => ApiClient;
-  createSSEManager?: (server: Server, apiKey: string) => SSEManager;
+  createSSEManager?: (server: Server) => SSEManager;
 };
 
 export class Server {
@@ -102,7 +102,7 @@ export class Server {
    * Uses custom ID if provided, otherwise falls back to URL-based ID.
    */
   get providerId(): string {
-    return this.options.customId ?? `${PROVIDER_PREFIX}=${this.baseUrl}`;
+    return ServerIds.resolve(this.baseUrl, this.options.customId);
   }
 
   /**
@@ -129,7 +129,7 @@ export class Server {
       if (key !== API_KEY_PLACEHOLDER) return key;
     }
     // Fall back to URL-based ID
-    return this.settings.resolveApiKey(`${PROVIDER_PREFIX}=${this.baseUrl}`);
+    return this.settings.resolveApiKey(ServerIds.fromUrl(this.baseUrl));
   }
 
   /**
@@ -141,9 +141,7 @@ export class Server {
     this.apiClient =
       this.deps.createApiClient?.(apiKey) ??
       new ApiClient(this.baseUrl, apiKey);
-    this.sse =
-      this.deps.createSSEManager?.(this, apiKey) ??
-      new SSEManager(this, apiKey);
+    this.sse = this.deps.createSSEManager?.(this) ?? new SSEManager(this);
     const { data } = await this.fetchModels();
     const mode = await this.detectServerMode(data);
 
@@ -190,15 +188,6 @@ export class Server {
     return checkServerHealth(this.baseUrl, timeout, this.getApiKey());
   }
 
-  /**
-   * Retrieves the health status of the server
-   *
-   * @returns The health status
-   */
-  async fetchServerHealth(): Promise<HealthEndpoint> {
-    return await this.apiClient.get<HealthEndpoint>("/health");
-  }
-
   /**
    * Fetches models from the server
    *
@@ -231,13 +220,6 @@ export class Server {
     );
   }
 
-  /**
-   * Returns the per-model override configuration for this server.
-   */
-  getOverrides(): Record<string, ModelOverride> {
-    return this.options.overrides ?? {};
-  }
-
   /**
    * Resolves the override for a given model ID using prefix matching.
    *
@@ -249,7 +231,7 @@ export class Server {
    * @returns The matching override, or `undefined` if no key matches.
    */
   findOverrideForModel(modelId: string): ModelOverride | undefined {
-    const overrides = this.getOverrides();
+    const overrides = this.options.overrides ?? {};
     let best: ModelOverride | undefined;
     let bestLen = 0;
 

package/src/sse/client.ts

@@ -1,30 +1,22 @@
 import { POLLING_INTERVAL } from "../constants";
+import { openSSEStream } from "./fetch";
 import type { SSECallback, SSECleanup, SSEEvent } from "./types";
 
-/**
- * Builds the full SSE endpoint URL, appending the API key as a query
- * parameter when one is set. Shared by {@link SSEClient} and
- * {@link SSEManager.probeSSE} so the two can't drift.
- */
-export const buildSSEUrl = (endpoint: string, apiKey?: string): string => {
-  if (apiKey) {
-    return `${endpoint}?api_key=${encodeURIComponent(apiKey)}`;
-  }
-  return endpoint;
-};
-
 /**
  * SSE client for llama-server's /models/sse endpoint.
  *
- * Uses a single shared EventSource per server instance.
+ * Uses a single shared stream per server instance, consumed with `fetch`
+ * (EventSource cannot send the API key as a header, and llama-server no
+ * longer accepts it as a query parameter — see `fetch.ts`).
  * Supports multiple model subscriptions with automatic event routing.
  * Handles reconnection by re-subscribing all callbacks.
  */
 export class SSEClient {
-  private eventSource: EventSource | null = null;
+  private abortController: AbortController | null = null;
+  private disposed: boolean = false;
   private subscribers: Map<string, SSECallback> = new Map();
   private connected: boolean = false;
-  private reconnecting: boolean = false; // tracks if EventSource auto-reconnect is in progress
+  private reconnecting: boolean = false; // tracks if reconnect is in progress
   /**
    * Single shared slot — each setOnConnectFailed call overwrites the
    * previous callback (see there for the constraint this imposes).
@@ -42,7 +34,15 @@ export class SSEClient {
   ) {}
 
   /**
-   * Connects to the SSE endpoint.
+   * Waits the given amount of time (used between reconnection attempts).
+   */
+  private delay(ms: number): Promise<void> {
+    return new Promise<void>((resolve) => setTimeout(resolve, ms));
+  }
+
+  /**
+   * Connects to the SSE endpoint and keeps it open, reconnecting with a
+   * fixed delay until `disconnect()` is called.
    *
    * No current caller consumes the result: `subscribe()` triggers the
    * connection without awaiting it, and connection failures before the
@@ -50,56 +50,110 @@ export class SSEClient {
    *
    * @returns true if the connection was established successfully
    */
-  async connect(): Promise<boolean> {
+  private async connect(): Promise<boolean> {
     if (this.connected) return true;
+    this.disposed = false;
 
-    const url = buildSSEUrl(this.sseEndpoint, this.apiKey);
+    // Loop-local so a stale, already-aborted controller from a previous
+    // connection can't be confused with the current one after a revival.
+    let abortController: AbortController | null = this.abortController;
 
-    try {
-      this.eventSource = new EventSource(url);
-    } catch {
-      this.connected = false;
-      return false;
-    }
+    while (!this.disposed) {
+      if (abortController?.signal.aborted) return false;
+      abortController = new AbortController();
+      this.abortController = abortController;
 
-    this.eventSource.onerror = () => {
-      // EventSource will auto-reconnect; we just track state
-      this.connected = false;
+      let body: ReadableStream<Uint8Array>;
+      try {
+        body = await openSSEStream(
+          this.sseEndpoint,
+          this.apiKey,
+          abortController.signal,
+        );
+      } catch {
+        if (this.disposed || abortController.signal.aborted) return false;
+        this.notifyConnectFailed();
+        this.reconnecting = true;
+        await this.delay(POLLING_INTERVAL);
+        continue;
+      }
+
+      this.connected = true;
+      this.reconnecting = false;
+
+      await this.consume(body);
+
+      if (this.disposed || abortController.signal.aborted) return false;
+      // Stream ended (server closed or network error): reconnect
       this.reconnecting = true;
+      await this.delay(POLLING_INTERVAL);
+    }
 
-      // Notify subscriber if connection fails before any event is received
-      if (!this._hasReceivedEvents && this._onConnectFailed) {
-        this._onConnectFailed();
-      }
-    };
+    return false;
+  }
 
-    this.eventSource.onmessage = (event: MessageEvent) => {
-      try {
-        const data = JSON.parse(event.data);
-        const sseEvent: SSEEvent = {
-          event: data.event ?? "unknown",
-          model: data.model ?? "*",
-          data: data.data,
-        };
-        this._hasReceivedEvents = true;
-        this.dispatch(sseEvent);
-      } catch {
-        // Invalid JSON, ignore
+  /**
+   * Reads the raw byte stream, parses the SSE framing and dispatches
+   * `data:` payloads. Resolves when the stream ends or errors.
+   */
+  private async consume(body: ReadableStream<Uint8Array>): Promise<void> {
+    const reader = body.getReader();
+    const decoder = new TextDecoder();
+    let buffer = "";
+
+    try {
+      for (;;) {
+        const { done, value } = await reader.read();
+        if (done) break;
+
+        buffer += decoder.decode(value, { stream: true });
+        let newlineIndex: number;
+        while ((newlineIndex = buffer.indexOf("\n")) !== -1) {
+          const line = buffer.slice(0, newlineIndex);
+          buffer = buffer.slice(newlineIndex + 1);
+          this.handleLine(line);
+        }
       }
-    };
+      // Flush a trailing data line that lacked its event-terminating newline
+      if (buffer) this.handleLine(buffer);
+    } catch {
+      // Stream aborted or network error — handled by the reconnect logic
+    } finally {
+      reader.releaseLock();
+    }
+  }
 
-    // Wait a bit for the connection to establish
-    await new Promise<void>((resolve) => {
-      const timeout = setTimeout(() => resolve(), POLLING_INTERVAL);
-      this.eventSource!.onopen = () => {
-        clearTimeout(timeout);
-        this.connected = true;
-        this.reconnecting = false;
-        resolve();
+  /**
+   * Handles a single SSE line, dispatching `data:` payloads as events.
+   */
+  private handleLine(line: string): void {
+    if (!line.startsWith("data:")) return;
+
+    const payload = line.slice(5).trim();
+    if (!payload || payload === "[DONE]") return;
+
+    try {
+      const data = JSON.parse(payload);
+      const sseEvent: SSEEvent = {
+        event: data.event ?? "unknown",
+        model: data.model ?? "*",
+        data: data.data,
       };
-    });
+      this._hasReceivedEvents = true;
+      this.dispatch(sseEvent);
+    } catch {
+      // Invalid JSON, ignore
+    }
+  }
 
-    return this.connected;
+  /**
+   * Notifies the connect-failed callback if the connection failed before
+   * any event was received.
+   */
+  private notifyConnectFailed(): void {
+    if (!this._hasReceivedEvents && this._onConnectFailed) {
+      this._onConnectFailed();
+    }
   }
 
   /**
@@ -143,11 +197,11 @@ export class SSEClient {
    * Disconnects from the SSE endpoint and clears all subscriptions.
    */
   disconnect(): void {
-    if (this.eventSource) {
-      this.eventSource.close();
-      this.eventSource = null;
-    }
+    this.disposed = true;
+    this.abortController?.abort();
+    this.abortController = null;
     this.connected = false;
+    this.reconnecting = false;
     this.subscribers.clear();
   }
 

package/src/sse/fetch.ts

@@ -0,0 +1,43 @@
+/**
+ * Auth and stream plumbing for llama-server's SSE endpoint.
+ *
+ * llama-server validates the API key from the `Authorization` (or
+ * `X-Api-Key`) header only — it no longer accepts it as a query parameter —
+ * and `EventSource` cannot send custom headers. The stream is therefore
+ * consumed with `fetch` and parsed manually by {@link SSEClient}.
+ *
+ * `buildAuthHeaders` is shared by {@link SSEClient} and
+ * {@link SSEManager.probeSSE} so the two can't drift.
+ */
+
+/**
+ * Builds the auth headers for a llama-server request, including the API key
+ * as an `Authorization: Bearer` header when one is set.
+ */
+export const buildAuthHeaders = (apiKey?: string): Record<string, string> =>
+  apiKey ? { Authorization: `Bearer ${apiKey}` } : {};
+
+/**
+ * Opens the SSE stream at the given endpoint, authenticating via headers.
+ *
+ * @returns the response body as a stream of raw bytes
+ * @throws if the request fails or the server responds with an error status
+ */
+export const openSSEStream = async (
+  endpoint: string,
+  apiKey?: string,
+  signal?: AbortSignal,
+): Promise<ReadableStream<Uint8Array>> => {
+  const response = await fetch(endpoint, {
+    method: "GET",
+    headers: buildAuthHeaders(apiKey),
+    signal,
+  });
+
+  if (!response.ok || !response.body) {
+    await response.body?.cancel();
+    throw new Error(`SSE connection failed: HTTP ${response.status}`);
+  }
+
+  return response.body;
+};