pi-llama-cpp 0.15.0 → 0.16.0

package/README.md

@@ -12,6 +12,7 @@ A [Pi Coding Agent](https://pi.dev/) extension that integrates with running [lla
 - **Flexible URL resolution** — configures the server via `llamaSettings` (project/global), environment variable, or legacy `llamaServerUrl`
 - **Auth support** — allows to login into a llama.cpp server that was secured with an API key
 - **Multiple server support** — connect to multiple llama.cpp servers simultaneously via `llamaSettings.servers` or semicolon-separated URLs
+- **Basic llama-swap support** — auto-detects and provides basic integration with [llama-swap](https://github.com/mostlygeek/llama-swap) gateways
 - **Thinking budget support** — configurable token budgets for model reasoning/thinking, mapped to Pi's thinking levels
 - **Real-time progress tracking** — live loading progress via SSE (falls back to polling)
 
@@ -207,17 +208,48 @@ Each server gets its own provider (e.g., **Llama.cpp (http://127.0.0.1:8080)**)
 If your llama.cpp server requires authentication, use `/login` in Pi, select the "API key" option, and choose the provider from the list that correlates with the server needing the API key.
 
 Alternatively, configure the API key in `~/.pi/agent/auth.json`:
-Use the provider ID `llama-server=<url>` (or your custom `id` if you set one in `llamaSettings.servers`):
+Use the provider ID `llama-server=<url>` (or your custom `id` if you set one in `llamaSettings.servers`).
+
+The `key` field supports several formats:
+
+| Format            | Example                                      | Description                                |
+| ----------------- | -------------------------------------------- | ------------------------------------------ |
+| **Literal**       | `"sk-abc123"`                                | API key stored directly                    |
+| **Env ref**       | `"$OPENAI_API_KEY"` or `"${OPENAI_API_KEY}"` | Resolved from `process.env` or `env` field |
+| **Shell command** | `"!cat ~/.secrets/llama-key"`                | Stdout of the command is used              |
+| **Escape**        | `"$$literal"`                                | `$$` → literal `$`, `$!` → literal `!`     |
 
 ```json
 {
   "llama-server=http://127.0.0.1:8080": {
     "type": "api_key",
-    "key": "<key-for-server-1>"
+    "key": "sk-abc123"
   },
   "llama-server=https://some-url-for-llama-cpp": {
     "type": "api_key",
-    "key": "<key-for-server-2>"
+    "key": "$LLAMA_API_KEY"
+  },
+  "llama-server=https://secure-server": {
+    "type": "api_key",
+    "key": "!cat ~/.secrets/llama-key"
+  },
+  "llama-server=https://braced-ref": {
+    "type": "api_key",
+    "key": "${API_KEY}"
+  }
+}
+```
+
+For env ref formats, you can also store the variable value alongside the key using the `env` field:
+
+```json
+{
+  "llama-server=http://127.0.0.1:8080": {
+    "type": "api_key",
+    "key": "$MY_KEY",
+    "env": {
+      "MY_KEY": "sk-abc123"
+    }
   }
 }
 ```
@@ -246,6 +278,10 @@ llama-server --model path/to/model.gguf ...
 
 > **Note:** The ik_llama.cpp fork is not legacy at all, but it uses an old way of describing models compared to llama.cpp.
 
+- For llama-swap mode, point the extension at a running [llama-swap](https://github.com/mostlygeek/llama-swap) instance instead of a raw llama.cpp server. The extension auto-detects this mode via the `src` field in the server props response.
+
+> **Note:** llama-swap support is basic — only model listing, status, load/unload, and capability detection are implemented.
+
 The extension determines the context size as follows:
 
 - A per-model `contextSize` override (see [Model Overrides](#model-overrides)) takes precedence over everything below
@@ -254,6 +290,7 @@ The extension determines the context size as follows:
   - When not loaded, reads `--ctx-size` and/or `--fit-ctx` from the server arguments (which can also originate from the **presets.ini** file the llama.cpp server uses to load its models).
 - **Single mode** — reads `meta.n_ctx` from the `/v1/models` endpoint
 - **Legacy mode** — reads `max_model_len` from `/v1/models`, falling back to `n_ctx` from `/props`
+- **Llama-swap mode** — reads `meta.n_ctx` from the llama-swap server via `/v1/models`
 - Falls back to `128000` if not available
 
 ### Commands

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-llama-cpp",
-  "version": "0.15.0",
+  "version": "0.16.0",
   "description": "Pi extension for llama.cpp integration. Supports router, single and legacy models. Supports multiple servers.",
   "keywords": [
     "pi",

package/src/api/client.ts

@@ -81,6 +81,30 @@ export class ApiClient {
     );
   }
 
+  /**
+   * Makes a raw GET request that does not parse JSON. Useful for endpoints
+   * that return empty or non-JSON responses.
+   *
+   * @param endpoint The endpoint path to fetch
+   */
+  async rawGet(endpoint: string): Promise<void> {
+    return this.do_request<void>(endpoint, "GET", undefined, false);
+  }
+
+  /**
+   * Makes a raw POST request that does not parse JSON. Useful for endpoints
+   * that return empty or non-JSON responses.
+   *
+   * @param endpoint The endpoint path to post to
+   * @param body The optional request body
+   */
+  async rawPost(
+    endpoint: string,
+    body?: Record<string, unknown>,
+  ): Promise<void> {
+    return this.do_request<void>(endpoint, "POST", body, false);
+  }
+
   /**
    * Clears the entire cache.
    */
@@ -134,26 +158,54 @@ export class ApiClient {
   }
 
   /**
-   * Makes a raw GET request to the llama-server.
+   * Makes a raw request to the llama-server.
    * This bypasses caching and deduplication.
    *
-   * @param endpoint The endpoint path to fetch (e.g. "/health")
-   * @returns The parsed JSON response from the server
+   * @param endpoint The endpoint path
+   * @param method The HTTP method
+   * @param body The optional request body
+   * @param parseJson Whether to parse the response as JSON (default: true)
+   * @returns The parsed JSON response, or void if parseJson is false
    * @throws ApiError with status 401 when the server rejects the request
    *   due to authentication (invalid/missing API key).
    */
-  private async do_get<T>(endpoint: string): Promise<T> {
+  private async do_request<T>(
+    endpoint: string,
+    method: "GET" | "POST",
+    body?: Record<string, unknown>,
+    parseJson = true,
+  ): Promise<T | void> {
     const url = `${this.baseUrl}${endpoint}`;
 
     const res = await fetch(url, {
-      headers: { Authorization: `Bearer ${this.apiKey}` },
+      method,
+      headers: {
+        ...(method === "POST"
+          ? { "Content-Type": "application/json" }
+          : undefined),
+        Authorization: `Bearer ${this.apiKey}`,
+      },
+      ...(body ? { body: JSON.stringify(body) } : undefined),
     });
 
     if (res.status === 401) {
       throw new ApiError("authentication", res.status);
     }
 
-    return res.json();
+    return parseJson ? ((await res.json()) as T) : (undefined as T);
+  }
+
+  /**
+   * Makes a raw GET request to the llama-server.
+   * This bypasses caching and deduplication.
+   *
+   * @param endpoint The endpoint path to fetch (e.g. "/health")
+   * @returns The parsed JSON response from the server
+   * @throws ApiError with status 401 when the server rejects the request
+   *   due to authentication (invalid/missing API key).
+   */
+  private async do_get<T>(endpoint: string): Promise<T> {
+    return this.do_request<T>(endpoint, "GET") as Promise<T>;
   }
 
   /**
@@ -170,21 +222,6 @@ export class ApiClient {
     endpoint: string,
     body?: Record<string, unknown>,
   ): Promise<T> {
-    const url = `${this.baseUrl}${endpoint}`;
-
-    const res = await fetch(url, {
-      method: "POST",
-      headers: {
-        "Content-Type": "application/json",
-        Authorization: `Bearer ${this.apiKey}`,
-      },
-      body: body ? JSON.stringify(body) : undefined,
-    });
-
-    if (res.status === 401) {
-      throw new ApiError("authentication", res.status);
-    }
-
-    return res.json();
+    return this.do_request<T>(endpoint, "POST", body) as Promise<T>;
   }
 }

package/src/enums/mode.ts

@@ -3,4 +3,5 @@ export enum Mode {
   ROUTER = "router",
   SINGLE = "single",
   LEGACY = "legacy",
+  LLAMASWAP = "llama-swap",
 }

package/src/interfaces/endpoints/models.ts

@@ -67,4 +67,10 @@ interface MetaProperty {
   n_embd: number;
   n_params: number;
   size: number;
+
+  // llama-swap extension
+  llamaswap?: {
+    aliases: string[];
+    type: string;
+  };
 }

package/src/interfaces/endpoints/props.ts

@@ -13,6 +13,20 @@ export interface PropsEndpoint {
   cors_proxy_enabled: boolean;
 }
 
+/**
+ * llama-swap returns a different shape for /props (without a model ID):
+ * an error response with a `src` that we can use as a trick to detect it.
+ */
+export interface LlamaSwapPropsError {
+  src: "llama-swap";
+  error: {
+    message: string;
+    type: string;
+    param: null;
+    code: string;
+  };
+}
+
 /**
  * The structure of llama-server's /props?model=<id> endpoint
  */

package/src/managers/command/models.ts

@@ -217,10 +217,9 @@ export class ModelsMenu {
     const base = [Action.INFO, Action.CANCEL];
 
     const actions: Record<Status, Array<Action>> = {
-      [Status.LOADED]:
-        model.mode === Mode.ROUTER
-          ? [Action.SWITCH, Action.UNLOAD, ...base]
-          : [Action.SWITCH, ...base],
+      [Status.LOADED]: [Mode.ROUTER, Mode.LLAMASWAP].includes(model.mode)
+        ? [Action.SWITCH, Action.UNLOAD, ...base]
+        : [Action.SWITCH, ...base],
       [Status.LOADING]: [...base],
       [Status.FAILED]: [Action.RETRY, ...base],
       [Status.SLEEPING]:

package/src/managers/settings.ts

@@ -24,6 +24,7 @@ import {
 } from "../interfaces/settings";
 import type { SortBy } from "../interfaces/sortBy";
 import { Server } from "../server";
+import { CredentialResolver } from "../utils/credentialResolver";
 import { SettingsStore } from "../utils/settingsStore";
 import { UrlResolver } from "../utils/urlResolver";
 
@@ -47,6 +48,9 @@ export class LlamaSettingsManager {
     }
   }
 
+  /** Delegates credential key resolution (see `utils/credentialResolver`). */
+  private credentialResolver = new CredentialResolver();
+
   /** Delegated multi-source URL resolution chain (see `utils/urlResolver`). */
   private urlResolver = new UrlResolver({
     getLlamaSettings: () => this.getLlamaSettings(),
@@ -149,12 +153,16 @@ export class LlamaSettingsManager {
 
   /**
    * Resolves API key for the provider ID using Pi's stored credentials.
+   * Delegates to `CredentialResolver` for key format handling.
    *
+   * @param providerId The provider ID
    * @returns The API key to use for the provider
    */
   resolveApiKey(providerId: string): string {
     const credential = readStoredCredential(providerId) as ApiKeyCredential;
-    return credential?.key ?? API_KEY_PLACEHOLDER;
+    if (!credential?.key) return API_KEY_PLACEHOLDER;
+
+    return this.credentialResolver.resolve(credential.key, credential.env);
   }
 
   /**

package/src/models/legacyModel.ts

@@ -3,7 +3,7 @@ import { Mode } from "../enums/mode";
 import { SingleModel } from "./singleModel";
 
 export class LegacyModel extends SingleModel {
-  get mode(): Mode {
+  override get mode(): Mode {
     return Mode.LEGACY;
   }
 
@@ -13,7 +13,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns The context size
    */
-  protected async getContextSize(): Promise<number> {
+  protected override async getContextSize(): Promise<number> {
     const props = await this.server.fetchModelProps(this.id);
     const models = await this.server.fetchModels();
 
@@ -34,7 +34,7 @@ export class LegacyModel extends SingleModel {
    *
    * @returns An array of capabilities, as expected by Pi
    */
-  protected async getCapabilities(): Promise<("text" | "image")[]> {
+  protected override async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/models/llamaSwapModel.ts

@@ -0,0 +1,90 @@
+import { FALLBACK_CTX } from "../constants";
+import { Mode } from "../enums/mode";
+import { Status } from "../enums/status";
+import { BaseModel } from "./baseModel";
+
+export class LlamaSwapModel extends BaseModel {
+  get mode(): Mode {
+    return Mode.LLAMASWAP;
+  }
+
+  override get name(): string {
+    // llama-swap adds a `llamaswap` key inside `meta` with aliases;
+    // fall back to the standard aliases array, then to id
+    return (
+      this.model.meta?.llamaswap?.aliases?.[0] ??
+      this.model.aliases?.[0] ??
+      this.model.id
+    );
+  }
+
+  /**
+   * Retrieves the context size of the model.
+   * An override's `contextSize` takes precedence; otherwise reads
+   * `meta.n_ctx` from the llama-swap `/v1/models` response,
+   * falling back to `FALLBACK_CTX`.
+   *
+   * @returns The context size
+   */
+  protected override async getContextSize(): Promise<number> {
+    const overridden = this.server.findOverrideForModel(this.id)?.contextSize;
+    if (overridden && overridden > 0) return overridden;
+
+    const { data } = await this.server.fetchModels();
+    const model = data.find((m) => m.id === this.id);
+
+    return model?.meta?.n_ctx ?? FALLBACK_CTX;
+  }
+
+  /**
+   * Detects the capabilities of the model.
+   * An override's `capabilities` fully replaces detection.
+   *
+   * @returns An array of capabilities, as expected by Pi
+   */
+  override async getCapabilities(): Promise<("text" | "image")[]> {
+    const overridden = this.server.findOverrideForModel(this.id)?.capabilities;
+    if (overridden) return overridden;
+
+    const { data } = await this.server.fetchModels();
+    const model = data.find((m) => m.id === this.id);
+
+    const inputModalities = model?.architecture?.input_modalities ?? [];
+    return inputModalities.includes("image") ? ["text", "image"] : ["text"];
+  }
+
+  /**
+   * Detects the load status of a model.
+   * For simplicity, we'll only handle loaded/unloaded
+   *
+   * @returns The current {@link Status}
+   */
+  override async getStatus(): Promise<Status> {
+    const { data } = await this.server.fetchModels();
+    const model = data.find((m) => m.id === this.id);
+
+    return model?.status?.value === "loaded" ? Status.LOADED : Status.UNLOADED;
+  }
+
+  /**
+   * Loads the model in the llama-swap server
+   */
+  override async load(): Promise<void> {
+    const status = await this.getStatus();
+    if (status === Status.LOADED) return;
+
+    try {
+      await this.server.llamaSwapLoad(this.id);
+    } catch (err) {
+      console.warn({ err });
+      throw new Error(`Model loading failed: ${this.id}`);
+    }
+  }
+
+  /**
+   * Unloads the model in the llama-swap server
+   */
+  override async unload(): Promise<void> {
+    await this.server.llamaSwapUnload(this.id);
+  }
+}

package/src/models/routerModel.ts

@@ -26,7 +26,7 @@ export class RouterModel extends BaseModel {
    *
    * In exchange, it will allow unloaded models to be correctly shown as "unloaded".
    */
-  protected async pollStatus(startTime = Date.now()): Promise<void> {
+  protected override async pollStatus(startTime = Date.now()): Promise<void> {
     let elapsed = 0;
     const limit = 5000;
 
@@ -52,7 +52,7 @@ export class RouterModel extends BaseModel {
    *
    * @returns The context size in tokens
    */
-  protected async getContextSize(): Promise<number> {
+  protected override async getContextSize(): Promise<number> {
     // We can get a more accurate context size if the model is already loaded
     if ((await this.getStatus()) === Status.LOADED) {
       return super.getContextSize();

package/src/models/singleModel.ts

@@ -2,11 +2,11 @@ import { Mode } from "../enums/mode";
 import { BaseModel } from "./baseModel";
 
 export class SingleModel extends BaseModel {
-  get mode(): Mode {
+  override get mode(): Mode {
     return Mode.SINGLE;
   }
 
-  protected async getCapabilities(): Promise<("text" | "image")[]> {
+  protected override async getCapabilities(): Promise<("text" | "image")[]> {
     try {
       return await super.getCapabilities();
     } catch {

package/src/server.ts

@@ -8,6 +8,7 @@ import { Mode } from "./enums/mode";
 import { ServerStatus } from "./enums/serverStatus";
 import { ModelsEndpoint } from "./interfaces/endpoints/models";
 import {
+  LlamaSwapPropsError,
   PropsEndpoint,
   PropsModelEndpoint,
 } from "./interfaces/endpoints/props";
@@ -16,6 +17,7 @@ import type { ModelOverride } from "./interfaces/settings";
 import type { LlamaSettingsManager } from "./managers/settings";
 import { BaseModel } from "./models/baseModel";
 import { LegacyModel } from "./models/legacyModel";
+import { LlamaSwapModel } from "./models/llamaSwapModel";
 import { RouterModel } from "./models/routerModel";
 import { SingleModel } from "./models/singleModel";
 import { SSEManager } from "./sse/manager";
@@ -147,6 +149,7 @@ export class Server {
       [Mode.ROUTER]: RouterModel,
       [Mode.LEGACY]: LegacyModel,
       [Mode.SINGLE]: SingleModel,
+      [Mode.LLAMASWAP]: LlamaSwapModel,
     }[mode];
 
     const models: BaseModel[] = data.map((m) => new modelCtor(m, this));
@@ -163,9 +166,12 @@ export class Server {
    * @returns The detected mode
    */
   private async detectServerMode(data: ModelsEndpoint["data"]): Promise<Mode> {
-    const { role } = await this.fetchServerProps();
+    const serverProps = await this.fetchServerProps();
 
-    if (role === "router") return Mode.ROUTER;
+    if ("src" in serverProps && serverProps.src === "llama-swap")
+      return Mode.LLAMASWAP;
+    if ("role" in serverProps && serverProps.role === "router")
+      return Mode.ROUTER;
     if ("max_model_len" in data[0]) return Mode.LEGACY;
     return Mode.SINGLE;
   }
@@ -197,12 +203,16 @@ export class Server {
   }
 
   /**
-   * Fetches general properties of the server
+   * Fetches general properties of the server.
+   * llama-server returns {@link PropsEndpoint}; llama-swap returns
+   * {@link LlamaSwapPropsError} (an error response with a `src` discriminator).
    *
-   * @return The properties of the server
+   * @return The server properties
    */
-  async fetchServerProps(): Promise<PropsEndpoint> {
-    return await this.apiClient.get<PropsEndpoint>("/props?autoload=false");
+  async fetchServerProps(): Promise<PropsEndpoint | LlamaSwapPropsError> {
+    return await this.apiClient.get<PropsEndpoint | LlamaSwapPropsError>(
+      "/props?autoload=false",
+    );
   }
 
   /**
@@ -258,4 +268,24 @@ export class Server {
       model,
     });
   }
+
+  /**
+   * Loads a model on a llama-swap server (GET /upstream/{id}).
+   *
+   * @param model The model ID to load
+   */
+  async llamaSwapLoad(model: string): Promise<void> {
+    this.apiClient.clearCache();
+    await this.apiClient.rawGet(`/upstream/${model}`);
+  }
+
+  /**
+   * Unloads a model on a llama-swap server (POST /api/models/unload/{id}).
+   *
+   * @param model The model ID to unload
+   */
+  async llamaSwapUnload(model: string): Promise<void> {
+    this.apiClient.clearCache();
+    await this.apiClient.rawPost(`/api/models/unload/${model}`);
+  }
 }

package/src/ui/dialog/confirm.ts

@@ -39,7 +39,7 @@ export class ConfirmDialog extends BaseDialog {
 
   // -- helpers -------------------------------------------------------------------
 
-  protected get inputTarget() {
+  protected override get inputTarget() {
     return this.list;
   }
 }

package/src/ui/dialog/input.ts

@@ -36,7 +36,7 @@ export class InputDialog extends BaseDialog {
 
   // -- helpers -------------------------------------------------------------------
 
-  protected get inputTarget() {
+  protected override get inputTarget() {
     return this.input;
   }
 

package/src/ui/editors/override/entryEditor.ts

@@ -31,13 +31,13 @@ export class OverrideEntryListEditor extends ListEditor<OverrideSettingsListOpti
 
   /** Esc steps back to the server list, not out of the whole dialog —
    * `options.done` is the top-level close (shared options object). */
-  protected close(): void {
+  protected override close(): void {
     this.done();
   }
 
   // -- abstract hooks -------------------------------------------------------
 
-  protected buildSettingsList(): SettingsList {
+  protected override buildSettingsList(): SettingsList {
     const builder = new OverrideItemBuilder(this.dialogs);
     const items: SettingItem[] = this.entries().map((entry, i) => ({
       id: `entry-${i}`,
@@ -103,7 +103,7 @@ export class OverrideEntryListEditor extends ListEditor<OverrideSettingsListOpti
     );
   }
 
-  protected beginAdd(): void {
+  protected override beginAdd(): void {
     this.openDialog(
       this.dialogs.input({
         title: TITLES.addOverride,
@@ -119,7 +119,7 @@ export class OverrideEntryListEditor extends ListEditor<OverrideSettingsListOpti
     );
   }
 
-  protected deleteSelected(): void {
+  protected override deleteSelected(): void {
     const idx = this.selectedIndex;
     const next = this.removeEntry(idx);
     void this.persistSnapshot(next, () => {
@@ -129,21 +129,21 @@ export class OverrideEntryListEditor extends ListEditor<OverrideSettingsListOpti
     });
   }
 
-  protected readonly emptyHintKey = "emptyOverrideEntries" as const;
+  protected override readonly emptyHintKey = "emptyOverrideEntries" as const;
 
-  protected getCount(): number {
+  protected override getCount(): number {
     return this.entries().length;
   }
 
-  protected getRowId(index: number): string {
+  protected override getRowId(index: number): string {
     return `entry-${index}`;
   }
 
-  protected getRowLabel(index: number): string {
+  protected override getRowLabel(index: number): string {
     return this.entries()[index]?.pattern ?? "";
   }
 
-  protected get deleteTitle(): string {
+  protected override get deleteTitle(): string {
     return TITLES.deleteOverride;
   }
 

package/src/ui/editors/override/itemBuilder.ts

@@ -12,7 +12,7 @@ export class OverrideItemBuilder extends ItemBuilder<
   OverrideEntry,
   OverrideField
 > {
-  protected get fields(): readonly OverrideField[] {
+  protected override get fields(): readonly OverrideField[] {
     return OverrideFields.all;
   }
 
@@ -21,7 +21,10 @@ export class OverrideItemBuilder extends ItemBuilder<
    * an `InputDialog` submenu (validating against the field definition)
    * for "input" fields.
    */
-  protected decorate(def: OverrideField, base: SettingItem): SettingItem {
+  protected override decorate(
+    def: OverrideField,
+    base: SettingItem,
+  ): SettingItem {
     if (def.type === "finite") {
       return { ...base, values: [...(def.options ?? [])] };
     }

package/src/ui/editors/server/fields.ts

@@ -26,7 +26,7 @@ class UrlField extends ServerField {
   readonly field = FIELDS.serverUrl;
   readonly validate = ServerUrl.parse;
 
-  currentValue(server: LlamaServer): string {
+  override currentValue(server: LlamaServer): string {
     return server.url;
   }
 
@@ -51,7 +51,7 @@ class OptionalTextField extends ServerField {
     this.field = field;
   }
 
-  currentValue(server: LlamaServer): string {
+  override currentValue(server: LlamaServer): string {
     return server[this.id] ?? "";
   }
 

package/src/ui/editors/server/itemBuilder.ts

@@ -10,7 +10,7 @@ import { ServerDisplay } from "./utils";
  * Builds `SettingItem` objects for server rows and their field-edit submenus.
  */
 export class ServerItemBuilder extends ItemBuilder<LlamaServer, ServerField> {
-  protected get fields(): readonly ServerField[] {
+  protected override get fields(): readonly ServerField[] {
     return ServerFields.all;
   }
 
@@ -18,7 +18,10 @@ export class ServerItemBuilder extends ItemBuilder<LlamaServer, ServerField> {
    * Adds the field-edit submenu: an `InputDialog` validating against the
    * field definition.
    */
-  protected decorate(def: ServerField, base: SettingItem): SettingItem {
+  protected override decorate(
+    def: ServerField,
+    base: SettingItem,
+  ): SettingItem {
     return {
       ...base,
       submenu: this.dialogs.inputSubmenu(