pi-llama-cpp 0.9.2 → 0.11.0

package/README.md

@@ -9,9 +9,9 @@ A [Pi Coding Agent](https://pi.dev/) extension that integrates with running [lla
 - **Load / unload / switch** — manage models directly from the Pi command palette
 - **Multi-model router support** — works with both single-model and multi-model llama.cpp server configurations
 - **Image capabilities detection** — detects multimodal models automatically
-- **Flexible URL resolution** — configures the server URL via project config, environment variable, or global settings
+- **Flexible URL resolution** — configures the server via `llamaSettings` (project/global), environment variable, or legacy `llamaServerUrl`
 - **Auth support** — allows to login into a llama.cpp server that was secured with an API key
-- **Multiple server support** — connect to multiple llama.cpp servers simultaneously by separating URLs with semicolons
+- **Multiple server support** — connect to multiple llama.cpp servers simultaneously via `llamaSettings.servers` or semicolon-separated URLs
 - **Thinking budget support** — configurable token budgets for model reasoning/thinking, mapped to Pi's thinking levels
 - **Real-time progress tracking** — live loading progress via SSE (falls back to polling)
 
@@ -48,34 +48,154 @@ pi install https://github.com/gsanhueza/pi-llama-cpp
 
 ## Configuration
 
-The extension resolves the llama.cpp server URL(s) using the following priority order:
+The extension resolves the llama.cpp server configuration using the following priority order:
 
-1. **Per-project config** — `.pi/settings.json` in your project root:
+1. **Environment variable** — `LLAMA_SERVER_URL`
+2. **`llamaSettings`** — Main configuration format in `.pi/settings.json` (project) or `~/.pi/agent/settings.json` (global)
+3. **`llamaServerUrl`** — Legacy format in `.pi/settings.json` (project) or `~/.pi/agent/settings.json` (global)
+4. **Default** — `http://127.0.0.1:8080`
+
+### Server configuration
 
-   ```json
-   {
-     "llamaServerUrl": "http://127.0.0.1:8080"
-   }
-   ```
+The recommended way to configure the extension is using the `llamaSettings` key. This provides a structured way to define multiple servers with custom names and IDs, plus additional behavior options.
 
-2. **Environment variable** — `LLAMA_SERVER_URL`
+Add this to your `.pi/settings.json` (project) or `~/.pi/agent/settings.json` (global):
 
-3. **Global settings** — `~/.pi/agent/settings.json`:
+```json
+{
+  "llamaSettings": {
+    "servers": [
+      {
+        "url": "http://127.0.0.1:8080",
+        "id": "local",
+        "name": "Local Server"
+      },
+      {
+        "url": "http://10.0.0.5:8080",
+        "name": "Remote Server"
+      }
+    ],
+    "reactToModelSelect": true,
+    "autoloadOnMessage": false,
+    "sortBy": "asc",
+    "pollingTimeout": 60000,
+    "serverTimeout": 1000
+  }
+}
+```
 
-   ```json
-   {
-     "llamaServerUrl": "http://127.0.0.1:8080"
-   }
-   ```
+With this config, the servers will appear in Pi as **Llama.cpp (Local Server)** and **Llama.cpp (Remote Server)**.
+
+#### Server Options
+
+| Option | Type   | Required | Description                                                                  |
+| ------ | ------ | -------- | ---------------------------------------------------------------------------- |
+| `url`  | string | Yes      | The URL of the llama.cpp server                                              |
+| `id`   | string | No       | Custom provider ID (used for API key auth). Defaults to `llama-server=<url>` |
+| `name` | string | No       | Display name for the server in the UI (shown as `Llama.cpp — <name>`)        |
+
+> **Note:** If you set a custom `id`, you can use it in `~/.pi/agent/auth.json`. The extension will also fall back to the URL-based ID if no key is found for the custom `id`.
+
+#### Settings Options
+
+| Option               | Type    | Default | Description                                                   |
+| -------------------- | ------- | ------- | ------------------------------------------------------------- |
+| `reactToModelSelect` | boolean | `true`  | Load the model when you switch via Pi's model picker.         |
+| `autoloadOnMessage`  | boolean | `false` | Automatically load an unloaded model before sending a message |
+| `sortBy`             | string  | `"asc"` | Sort order for models (see below)                             |
+| `pollingTimeout`     | number  | `60000` | Max time (ms) to wait for model loading before giving up      |
+| `serverTimeout`      | number  | `1000`  | Timeout (ms) for server health checks and SSE probes          |
+
+> **Note:** `serverTimeout` controls individual HTTP request timeouts (health checks, SSE probe). `pollingTimeout` controls the total wait time for a model to finish loading. Increase `serverTimeout` for slow/high-latency servers, and `pollingTimeout` for large models or slow hardware.
+
+#### In-session settings menu
+
+Run `/models settings` to edit the scalar settings above without hand-editing JSON:
+
+- **Enter/Space** cycles the value under the cursor; **Esc** closes the menu.
+- Booleans toggle `on`/`off`, `sortBy` cycles through the sort orders, and the
+  timeouts cycle through presets (`pollingTimeout`: 15s/30s/60s/120s/300s,
+  `serverTimeout`: 500ms/1s/2s/5s/10s).
+- Changes are written to the **global** `~/.pi/agent/settings.json` only. If a
+  project `.pi/settings.json` defines the same key, its value keeps winning in
+  the merged view until you remove it there.
+- Boolean and sort changes apply immediately; timeout changes apply on the next
+  model load.
+- The `servers` list is edited with `/models servers` (see below).
+
+#### Server list editor
+
+Run `/models servers` to add, edit or remove entries of `llamaSettings.servers`
+without hand-editing JSON:
+
+- **↑/↓** moves the cursor, **Enter/e** edits the selected URL, **i** edits
+  its `id`, **n** its `name`, **a** adds a new entry, **d** deletes it
+  (after an "Are you sure?" confirmation — only **y** confirms;
+  **Enter** is ignored, **Esc/n** cancels), **Esc** closes the editor.
+- One URL per entry (`http://host:port`). Trailing slashes are stripped on
+  save; `;`-separated values are rejected — use separate entries instead.
+- Each change is written immediately to the **global**
+  `~/.pi/agent/settings.json`. If a project `.pi/settings.json` defines
+  `servers`, its list keeps winning in the merged view until you remove it
+  there.
+- Changes apply the next time providers are scanned — run `/models` to see
+  them. Additions, removals, and URL/`id`/`name` edits all take effect on
+  the next `/models`: new servers register their providers, removed ones
+  leave pi's registry immediately, and edited ones are re-registered with
+  the fresh config — no restart needed.
+- Limitation: a model already loading in the background on a removed or
+  edited server finishes loading, but its progress notifications stop;
+  re-select it from the (new) provider afterwards.
+- The editor shows a warning when the `LLAMA_SERVER_URL` environment variable
+  is set, since it overrides the configured servers.
+- Per-server `id`/`name` overrides can be edited with **i**/**n**; saving an
+  empty value clears the override. The list shows them as a
+  `(<id> - <name>)` suffix, falling back to the auto-detected
+  `llama-server=<url>` id when no custom `id` is set.
+
+#### Environment variable
+
+For a quick setup, you can use the `LLAMA_SERVER_URL` environment variable instead of the JSON config:
 
-4. **Default** — `http://127.0.0.1:8080`
+```bash
+export LLAMA_SERVER_URL="http://127.0.0.1:8080"
+```
+
+This is equivalent to defining a single server in `llamaSettings.servers` with just a URL.
+
+### Legacy configuration
 
-### Multiple Servers
+For a simpler setup, you can use the legacy `llamaServerUrl` key:
 
-To connect to multiple llama.cpp servers simultaneously, add your URLs as a single string **separated with semicolons** in any of the examples above:
+```json
+{
+  "llamaServerUrl": "http://127.0.0.1:8080"
+}
+```
+
+This is equivalent to defining a single server in `llamaSettings.servers` with just a URL.
+
+### Multiple servers
+
+To connect to multiple llama.cpp servers simultaneously:
+
+**Using `llamaSettings` (recommended):**
+
+```json
+{
+  "llamaSettings": {
+    "servers": [
+      { "url": "http://127.0.0.1:8080" },
+      { "url": "http://127.0.0.1:8081" },
+      { "url": "http://10.0.0.5:8080" }
+    ]
+  }
+}
+```
+
+**Using the environment variable:**
 
 ```bash
-# Example for env, but you can use any of the other methods
 LLAMA_SERVER_URL="http://127.0.0.1:8080;http://127.0.0.1:8081;http://10.0.0.5:8080"
 ```
 
@@ -86,7 +206,7 @@ Each server gets its own provider (e.g., **Llama.cpp (http://127.0.0.1:8080)**)
 If your llama.cpp server requires authentication, use `/login` in Pi, select the "API key" option, and choose the provider from the list that correlates with the server needing the API key.
 
 Alternatively, configure the API key in `~/.pi/agent/auth.json`:
-Use the provider ID `llama-server=<url>`:
+Use the provider ID `llama-server=<url>` (or your custom `id` if you set one in `llamaSettings.servers`):
 
 ```json
 {
@@ -136,11 +256,13 @@ The extension determines the context size as follows:
 
 ### Commands
 
-| Command          | Description                                                                        |
-| ---------------- | ---------------------------------------------------------------------------------- |
-| `/models`        | Browse your models with live status. Select a model to load, switch, or unload it. |
-| `/models info`   | Show detailed information for all available models at once.                        |
-| `/models unload` | Unload all loaded models at once.                                                  |
+| Command            | Description                                                                        |
+| ------------------ | ---------------------------------------------------------------------------------- |
+| `/models`          | Browse your models with live status. Select a model to load, switch, or unload it. |
+| `/models info`     | Show detailed information for all available models at once.                        |
+| `/models unload`   | Unload all loaded models at once.                                                  |
+| `/models servers`  | Add, edit or remove llama.cpp server URLs via a TUI editor.                        |
+| `/models settings` | Open a menu to edit the scalar `llamaSettings` fields.                             |
 
 > **Note:** When a llama.cpp server is slow to respond, it will be skipped at startup with a warning. Run `/models` to retry without timeout and see all models.
 
@@ -148,6 +270,18 @@ The extension determines the context size as follows:
 
 > **Note:** The `/models unload` command only makes sense in router mode.
 
+#### Model sorting
+
+The order of models in the `/models` menu is controlled by the `sortBy` setting:
+
+| Value         | Description                                                                             |
+| ------------- | --------------------------------------------------------------------------------------- |
+| `"asc"`       | Sort by model ID ascending (default)                                                    |
+| `"desc"`      | Sort by model ID descending                                                             |
+| `"asc-name"`  | Sort by model name ascending (ties broken by ID)                                        |
+| `"desc-name"` | Sort by model name descending (ties broken by ID)                                       |
+| `"api"`       | No sorting — models appear in the order returned by each server's `/v1/models` endpoint |
+
 ### Model Actions
 
 When browsing models via the `/models` command, you can:
@@ -199,13 +333,15 @@ When you switch models via Pi's model picker (instead of using the `/models` com
 
 This keeps the server in sync with the active model in Pi, regardless of how the switch was initiated — you don't need to manually load models before using them.
 
+You can disable this behavior by setting `reactToModelSelect` to `false` in `llamaSettings`.
+
 > **Note:** If you switch sessions while a model load is in-flight, you'll see a warning, but the load continues in the background. Use `/models` in the new session to verify the model status.
 
 ### Loading Models
 
 When you trigger a load, switch, or retry action, the extension uses SSE (Server-Sent Events) to receive real-time progress updates from the server. If SSE is not available, it falls back to polling.
 
-If loading takes longer than **60 seconds**, the operation times out with an error.
+If loading takes longer than **60 seconds** (configurable via `pollingTimeout`), the operation times out with an error.
 
 > **Note:** The timeout only applies to the progress detection. The model might still be loading in the background.
 

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-llama-cpp",
-  "version": "0.9.2",
+  "version": "0.11.0",
   "description": "Pi extension for llama.cpp integration. Supports router, single and legacy models. Supports multiple servers.",
   "keywords": [
     "pi",
@@ -36,9 +36,10 @@
     "@earendil-works/pi-coding-agent": ">=0.84.0",
     "@earendil-works/pi-tui": ">=0.84.0"
   },
+  "type": "module",
   "devDependencies": {
-    "@types/node": "^26.2.0",
+    "@types/node": "^26.4.1",
     "prettier-plugin-organize-imports": "^4.3.0",
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   }
 }

package/src/api/client.ts

@@ -1,13 +1,29 @@
 import { POLLING_INTERVAL } from "../constants";
-import { Cache } from "../utils/cache";
-import { Mutex } from "../utils/mutex";
 
 /**
- * HTTP client for llama-server with caching and deduplication.
+ * How long GET responses stay cached: half the polling interval, so a poll
+ * tick always reaches the server while multiple reads within one tick
+ * (fan-outs like `toProviderConfig`, concurrent model polls) do not.
+ */
+const CACHE_TTL = POLLING_INTERVAL / 2;
+
+/**
+ * HTTP client for llama-server with GET caching and request deduplication.
+ *
+ * Two complementary mechanisms keep repeated reads from reaching the server:
+ * - the TTL cache absorbs time-spaced repeats (poll loops, sequential reads);
+ * - the in-flight map absorbs simultaneous bursts: concurrent callers for the
+ *   same key share one request's promise.
+ *
+ * POST requests are deduplicated but never cached — they are not idempotent
+ * reads, and the only caller (`Server.postRequest()`) clears the cache around
+ * them. The dedup matters: independent load triggers (command, auto-load,
+ * model-select) can race, and merging their duplicate `POST /models/load`
+ * into one server call avoids load-state glitches.
  */
 export class ApiClient {
-  private cache = new Cache(POLLING_INTERVAL / 2);
-  private mutex = new Mutex();
+  private cache = new Map<string, { data: unknown; timestamp: number }>();
+  private inflight = new Map<string, Promise<unknown>>();
 
   /**
    * Creates a new ApiClient.
@@ -22,40 +38,34 @@ export class ApiClient {
 
   /**
    * Makes a cached, deduplicated GET request to the llama-server.
-   * Results are cached for half the polling interval and in-flight requests are deduplicated.
    *
    * @param endpoint The endpoint path to fetch (e.g. "/health")
    * @returns The parsed JSON response from the server
    */
   async get<T>(endpoint: string): Promise<T> {
-    const cached = this.cache.get<T>(endpoint);
+    const cached = this.cacheGet<T>(endpoint);
     if (cached !== undefined) return cached;
 
-    return this.mutex.getOrCreate(endpoint, async () => {
-      const data = (await this.do_get<T>(endpoint)) as T;
-      this.cache.set(endpoint, data);
+    return this.dedupe(endpoint, async () => {
+      const data = await this.do_get<T>(endpoint);
+      this.cacheSet(endpoint, data);
       return data;
     });
   }
 
   /**
-   * Makes a cached, deduplicated POST request to the llama-server.
-   * Results are cached for half the polling interval and in-flight requests are deduplicated.
+   * Makes a deduplicated POST request to the llama-server.
+   * Concurrent duplicate requests share one server call; responses are
+   * never cached.
    *
    * @param endpoint The endpoint path to post to
    * @param body The optional request body
    * @returns The parsed JSON response from the server
    */
   async post<T>(endpoint: string, body?: Record<string, unknown>): Promise<T> {
-    const key = this.cacheKey(endpoint, body);
-    const cached = this.cache.get<T>(key);
-    if (cached !== undefined) return cached;
-
-    return this.mutex.getOrCreate(key, async () => {
-      const data = (await this.do_post<T>(endpoint, body)) as T;
-      this.cache.set(key, data);
-      return data;
-    });
+    return this.dedupe(this.cacheKey(endpoint, body), async () =>
+      this.do_post<T>(endpoint, body),
+    );
   }
 
   /**
@@ -65,6 +75,51 @@ export class ApiClient {
     this.cache.clear();
   }
 
+  /**
+   * Runs `fn` for the given key, or returns an existing in-flight promise.
+   * Concurrent callers for the same key share the same promise.
+   */
+  private dedupe<T>(key: string, fn: () => Promise<T>): Promise<T> {
+    const existing = this.inflight.get(key);
+    if (existing) return existing as Promise<T>;
+
+    const promise = fn().finally(() => {
+      this.inflight.delete(key);
+    });
+    this.inflight.set(key, promise);
+    return promise;
+  }
+
+  /**
+   * Gets a cached GET response. Returns `undefined` if missing or expired.
+   */
+  private cacheGet<T>(key: string): T | undefined {
+    const entry = this.cache.get(key);
+    if (!entry) return undefined;
+    if (Date.now() - entry.timestamp > CACHE_TTL) {
+      this.cache.delete(key);
+      return undefined;
+    }
+    return entry.data as T;
+  }
+
+  /**
+   * Stores a GET response in the cache with the current timestamp.
+   */
+  private cacheSet(key: string, data: unknown): void {
+    this.cache.set(key, { data, timestamp: Date.now() });
+  }
+
+  /**
+   * Builds the dedup key for a POST request.
+   *
+   * @param endpoint The endpoint path to post to
+   * @param body The optional request body
+   */
+  private cacheKey(endpoint: string, body?: Record<string, unknown>): string {
+    return body ? `${endpoint}:${JSON.stringify(body)}` : endpoint;
+  }
+
   /**
    * Makes a raw GET request to the llama-server.
    * This bypasses caching and deduplication.
@@ -107,15 +162,4 @@ export class ApiClient {
 
     return res.json();
   }
-
-  /**
-   * Sets a cache key
-   *
-   * @param endpoint The endpoint path to post to
-   * @param body The optional request body
-   * @returns The cache key
-   */
-  private cacheKey(endpoint: string, body?: Record<string, unknown>): string {
-    return body ? `${endpoint}:${JSON.stringify(body)}` : endpoint;
-  }
 }

package/src/constants.ts

@@ -3,6 +3,11 @@
  */
 export const PROVIDER_PREFIX = "llama-server";
 
+/**
+ * The settings key used in project/global settings.
+ */
+export const SETTINGS_KEY = "llamaSettings";
+
 /**
  * This provider's name
  */
@@ -21,12 +26,12 @@ export const API_KEY_PLACEHOLDER = "sk-placeholder";
 /**
  * The default URL if the resolver couldn't find it
  */
-export const DEFAULT_LLAMA_SERVER_URL = "http://127.0.0.1:8080";
+export const LLAMA_SERVER_URL = "http://127.0.0.1:8080";
 
 /**
  * The default context if the server didn't expose it
  */
-export const DEFAULT_CTX = 128000;
+export const FALLBACK_CTX = 128000;
 
 /**
  * Polling interval (ms) for checking model load status
@@ -48,10 +53,30 @@ export const READABLE_TIMEOUT = 15000;
  */
 export const SERVER_TIMEOUT = 1000;
 
+/**
+ * Default value for reactToModelSelect setting.
+ */
+export const REACT_TO_MODEL_SELECT = true;
+
+/**
+ * Default value for autoloadOnMessage setting.
+ */
+export const AUTOLOAD_ON_MESSAGE = false;
+
+/**
+ * Default sort order for model lists.
+ */
+export const SORT_BY = "asc";
+
+/**
+ * Sort order options for model lists.
+ */
+export type SortBy = "asc" | "desc" | "asc-name" | "desc-name" | "api";
+
 /**
  * Thinking budgets to send to the server, depending on user-selected level in Pi.
  */
-export const DEFAULT_THINKING_BUDGETS = {
+export const THINKING_BUDGETS = {
   off: 0,
   minimal: 1024,
   low: 2048,

package/src/index.ts

@@ -11,17 +11,12 @@ import { ModelSelectEvent } from "./interfaces/events";
 import { CommandManager } from "./managers/command";
 import { EventManager } from "./managers/events";
 import { ServerManager } from "./managers/server";
-import { ConfigResolver } from "./resolver";
-import { Server } from "./server";
+import { settings } from "./managers/settings";
 
 export default async function (pi: ExtensionAPI) {
-  const resolver = new ConfigResolver();
-  const urls = await resolver.resolveUrls();
-  const servers = urls.map((url) => new Server(url));
-
-  const eventManager = new EventManager(servers);
-  const serverManager = new ServerManager(servers);
-  const commandManager = new CommandManager(serverManager);
+  const serverManager = new ServerManager(settings);
+  const eventManager = new EventManager(serverManager, settings);
+  const commandManager = new CommandManager(serverManager, settings);
 
   // Register providers once at startup
   await serverManager.initialize(pi);
@@ -40,9 +35,6 @@ export default async function (pi: ExtensionAPI) {
     if (event.reason !== "startup") return;
     for (const warning of serverManager.getWarnings())
       ctx.ui.notify(warning, "warning");
-
-    for (const warning of resolver.getWarnings())
-      ctx.ui.notify(warning, "warning");
   });
 
   pi.on(

package/src/interfaces/events.ts

@@ -1,3 +1,14 @@
-export interface ModelSelectEvent {
-  model: { id: string; provider: string };
-}
+import type { ExtensionEvent } from "@earendil-works/pi-coding-agent";
+
+/**
+ * pi-coding-agent does not re-export its `ModelSelectEvent` from the public
+ * API (it exists in the internal extensions/index but is omitted from the
+ * root re-export list, and the package's `exports` map blocks deep imports).
+ * It is therefore derived from the exported `ExtensionEvent` union, which
+ * yields pi's real event shape (`model: Model<any>`, `previousModel`,
+ * `source`) and tracks the API automatically.
+ */
+export type ModelSelectEvent = Extract<
+  ExtensionEvent,
+  { type: "model_select" }
+>;

package/src/interfaces/server.ts

@@ -0,0 +1,20 @@
+/**
+ * Identity of a llama.cpp server endpoint.
+ *
+ * Persisted counterpart: {@link LlamaServer} (`llamaSettings.servers`),
+ * whose `url`/`id`/`name` map onto these fields.
+ */
+export interface ServerOptions {
+  /**
+   * Base URL of the llama.cpp server (e.g. "http://127.0.0.1:8080").
+   */
+  baseUrl: string;
+  /**
+   * Custom provider ID; falls back to a URL-based one.
+   */
+  customId?: string;
+  /**
+   * Custom provider name suffix; falls back to the base URL.
+   */
+  customName?: string;
+}

package/src/interfaces/settings.ts

@@ -0,0 +1,69 @@
+import type { SortBy } from "../constants";
+
+/**
+ * A description of a server in the "llamaSettings" key
+ */
+export interface LlamaServer {
+  /**
+   * The URL of the llama.cpp server.
+   */
+  url: string;
+  /**
+   * Custom provider ID for this server.
+   */
+  id?: string;
+  /**
+   * Custom display name for this server.
+   */
+  name?: string;
+}
+
+/**
+ * The main configuration interface for this extension
+ *
+ * E.g.:
+ *
+ * {
+ *   "servers": [{
+ *     "id": "server-a",
+ *     "name": "Server A",
+ *     "url": "http://localhost:8080"
+ *   }],
+ *   "reactToModelSelect": true
+ *   "autoloadOnMessage": false
+ *   "sortBy": "asc"
+ * }
+}
+ */
+export interface LlamaSettings {
+  /**
+   * List of servers to connect to.
+   * @default []
+   */
+  servers?: LlamaServer[];
+  /**
+   * Whether to react to model selection events by loading the model.
+   * @default true
+   */
+  reactToModelSelect?: boolean;
+  /**
+   * Whether to auto-load models when a message is sent.
+   * @default false
+   */
+  autoloadOnMessage?: boolean;
+  /**
+   * Maximum time (ms) to wait for model loading before giving up.
+   * @default 60000
+   */
+  pollingTimeout?: number;
+  /**
+   * Timeout (ms) for server verification and SSE support probe.
+   * @default 1000
+   */
+  serverTimeout?: number;
+  /**
+   * How to sort models in the /models command.
+   * @default "asc"
+   */
+  sortBy?: SortBy;
+}