pi-llama-cpp 0.10.0 → 0.11.0

package/README.md

@@ -77,6 +77,7 @@ Add this to your `.pi/settings.json` (project) or `~/.pi/agent/settings.json` (g
     ],
     "reactToModelSelect": true,
     "autoloadOnMessage": false,
+    "sortBy": "asc",
     "pollingTimeout": 60000,
     "serverTimeout": 1000
   }
@@ -101,11 +102,57 @@ With this config, the servers will appear in Pi as **Llama.cpp (Local Server)**
 | -------------------- | ------- | ------- | ------------------------------------------------------------- |
 | `reactToModelSelect` | boolean | `true`  | Load the model when you switch via Pi's model picker.         |
 | `autoloadOnMessage`  | boolean | `false` | Automatically load an unloaded model before sending a message |
+| `sortBy`             | string  | `"asc"` | Sort order for models (see below)                             |
 | `pollingTimeout`     | number  | `60000` | Max time (ms) to wait for model loading before giving up      |
 | `serverTimeout`      | number  | `1000`  | Timeout (ms) for server health checks and SSE probes          |
 
 > **Note:** `serverTimeout` controls individual HTTP request timeouts (health checks, SSE probe). `pollingTimeout` controls the total wait time for a model to finish loading. Increase `serverTimeout` for slow/high-latency servers, and `pollingTimeout` for large models or slow hardware.
 
+#### In-session settings menu
+
+Run `/models settings` to edit the scalar settings above without hand-editing JSON:
+
+- **Enter/Space** cycles the value under the cursor; **Esc** closes the menu.
+- Booleans toggle `on`/`off`, `sortBy` cycles through the sort orders, and the
+  timeouts cycle through presets (`pollingTimeout`: 15s/30s/60s/120s/300s,
+  `serverTimeout`: 500ms/1s/2s/5s/10s).
+- Changes are written to the **global** `~/.pi/agent/settings.json` only. If a
+  project `.pi/settings.json` defines the same key, its value keeps winning in
+  the merged view until you remove it there.
+- Boolean and sort changes apply immediately; timeout changes apply on the next
+  model load.
+- The `servers` list is edited with `/models servers` (see below).
+
+#### Server list editor
+
+Run `/models servers` to add, edit or remove entries of `llamaSettings.servers`
+without hand-editing JSON:
+
+- **↑/↓** moves the cursor, **Enter/e** edits the selected URL, **i** edits
+  its `id`, **n** its `name`, **a** adds a new entry, **d** deletes it
+  (after an "Are you sure?" confirmation — only **y** confirms;
+  **Enter** is ignored, **Esc/n** cancels), **Esc** closes the editor.
+- One URL per entry (`http://host:port`). Trailing slashes are stripped on
+  save; `;`-separated values are rejected — use separate entries instead.
+- Each change is written immediately to the **global**
+  `~/.pi/agent/settings.json`. If a project `.pi/settings.json` defines
+  `servers`, its list keeps winning in the merged view until you remove it
+  there.
+- Changes apply the next time providers are scanned — run `/models` to see
+  them. Additions, removals, and URL/`id`/`name` edits all take effect on
+  the next `/models`: new servers register their providers, removed ones
+  leave pi's registry immediately, and edited ones are re-registered with
+  the fresh config — no restart needed.
+- Limitation: a model already loading in the background on a removed or
+  edited server finishes loading, but its progress notifications stop;
+  re-select it from the (new) provider afterwards.
+- The editor shows a warning when the `LLAMA_SERVER_URL` environment variable
+  is set, since it overrides the configured servers.
+- Per-server `id`/`name` overrides can be edited with **i**/**n**; saving an
+  empty value clears the override. The list shows them as a
+  `(<id> - <name>)` suffix, falling back to the auto-detected
+  `llama-server=<url>` id when no custom `id` is set.
+
 #### Environment variable
 
 For a quick setup, you can use the `LLAMA_SERVER_URL` environment variable instead of the JSON config:
@@ -209,11 +256,13 @@ The extension determines the context size as follows:
 
 ### Commands
 
-| Command          | Description                                                                        |
-| ---------------- | ---------------------------------------------------------------------------------- |
-| `/models`        | Browse your models with live status. Select a model to load, switch, or unload it. |
-| `/models info`   | Show detailed information for all available models at once.                        |
-| `/models unload` | Unload all loaded models at once.                                                  |
+| Command            | Description                                                                        |
+| ------------------ | ---------------------------------------------------------------------------------- |
+| `/models`          | Browse your models with live status. Select a model to load, switch, or unload it. |
+| `/models info`     | Show detailed information for all available models at once.                        |
+| `/models unload`   | Unload all loaded models at once.                                                  |
+| `/models servers`  | Add, edit or remove llama.cpp server URLs via a TUI editor.                        |
+| `/models settings` | Open a menu to edit the scalar `llamaSettings` fields.                             |
 
 > **Note:** When a llama.cpp server is slow to respond, it will be skipped at startup with a warning. Run `/models` to retry without timeout and see all models.
 
@@ -221,6 +270,18 @@ The extension determines the context size as follows:
 
 > **Note:** The `/models unload` command only makes sense in router mode.
 
+#### Model sorting
+
+The order of models in the `/models` menu is controlled by the `sortBy` setting:
+
+| Value         | Description                                                                             |
+| ------------- | --------------------------------------------------------------------------------------- |
+| `"asc"`       | Sort by model ID ascending (default)                                                    |
+| `"desc"`      | Sort by model ID descending                                                             |
+| `"asc-name"`  | Sort by model name ascending (ties broken by ID)                                        |
+| `"desc-name"` | Sort by model name descending (ties broken by ID)                                       |
+| `"api"`       | No sorting — models appear in the order returned by each server's `/v1/models` endpoint |
+
 ### Model Actions
 
 When browsing models via the `/models` command, you can:

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-llama-cpp",
-  "version": "0.10.0",
+  "version": "0.11.0",
   "description": "Pi extension for llama.cpp integration. Supports router, single and legacy models. Supports multiple servers.",
   "keywords": [
     "pi",
@@ -38,8 +38,8 @@
   },
   "type": "module",
   "devDependencies": {
-    "@types/node": "^26.4.0",
+    "@types/node": "^26.4.1",
     "prettier-plugin-organize-imports": "^4.3.0",
-    "vitest": "^4.1.11"
+    "vitest": "^5.0.0"
   }
 }

package/src/api/client.ts

@@ -1,13 +1,29 @@
 import { POLLING_INTERVAL } from "../constants";
-import { Cache } from "../utils/cache";
-import { Mutex } from "../utils/mutex";
 
 /**
- * HTTP client for llama-server with caching and deduplication.
+ * How long GET responses stay cached: half the polling interval, so a poll
+ * tick always reaches the server while multiple reads within one tick
+ * (fan-outs like `toProviderConfig`, concurrent model polls) do not.
+ */
+const CACHE_TTL = POLLING_INTERVAL / 2;
+
+/**
+ * HTTP client for llama-server with GET caching and request deduplication.
+ *
+ * Two complementary mechanisms keep repeated reads from reaching the server:
+ * - the TTL cache absorbs time-spaced repeats (poll loops, sequential reads);
+ * - the in-flight map absorbs simultaneous bursts: concurrent callers for the
+ *   same key share one request's promise.
+ *
+ * POST requests are deduplicated but never cached — they are not idempotent
+ * reads, and the only caller (`Server.postRequest()`) clears the cache around
+ * them. The dedup matters: independent load triggers (command, auto-load,
+ * model-select) can race, and merging their duplicate `POST /models/load`
+ * into one server call avoids load-state glitches.
  */
 export class ApiClient {
-  private cache = new Cache(POLLING_INTERVAL / 2);
-  private mutex = new Mutex();
+  private cache = new Map<string, { data: unknown; timestamp: number }>();
+  private inflight = new Map<string, Promise<unknown>>();
 
   /**
    * Creates a new ApiClient.
@@ -22,40 +38,34 @@ export class ApiClient {
 
   /**
    * Makes a cached, deduplicated GET request to the llama-server.
-   * Results are cached for half the polling interval and in-flight requests are deduplicated.
    *
    * @param endpoint The endpoint path to fetch (e.g. "/health")
    * @returns The parsed JSON response from the server
    */
   async get<T>(endpoint: string): Promise<T> {
-    const cached = this.cache.get<T>(endpoint);
+    const cached = this.cacheGet<T>(endpoint);
     if (cached !== undefined) return cached;
 
-    return this.mutex.getOrCreate(endpoint, async () => {
-      const data = (await this.do_get<T>(endpoint)) as T;
-      this.cache.set(endpoint, data);
+    return this.dedupe(endpoint, async () => {
+      const data = await this.do_get<T>(endpoint);
+      this.cacheSet(endpoint, data);
       return data;
     });
   }
 
   /**
-   * Makes a cached, deduplicated POST request to the llama-server.
-   * Results are cached for half the polling interval and in-flight requests are deduplicated.
+   * Makes a deduplicated POST request to the llama-server.
+   * Concurrent duplicate requests share one server call; responses are
+   * never cached.
    *
    * @param endpoint The endpoint path to post to
    * @param body The optional request body
    * @returns The parsed JSON response from the server
    */
   async post<T>(endpoint: string, body?: Record<string, unknown>): Promise<T> {
-    const key = this.cacheKey(endpoint, body);
-    const cached = this.cache.get<T>(key);
-    if (cached !== undefined) return cached;
-
-    return this.mutex.getOrCreate(key, async () => {
-      const data = (await this.do_post<T>(endpoint, body)) as T;
-      this.cache.set(key, data);
-      return data;
-    });
+    return this.dedupe(this.cacheKey(endpoint, body), async () =>
+      this.do_post<T>(endpoint, body),
+    );
   }
 
   /**
@@ -65,6 +75,51 @@ export class ApiClient {
     this.cache.clear();
   }
 
+  /**
+   * Runs `fn` for the given key, or returns an existing in-flight promise.
+   * Concurrent callers for the same key share the same promise.
+   */
+  private dedupe<T>(key: string, fn: () => Promise<T>): Promise<T> {
+    const existing = this.inflight.get(key);
+    if (existing) return existing as Promise<T>;
+
+    const promise = fn().finally(() => {
+      this.inflight.delete(key);
+    });
+    this.inflight.set(key, promise);
+    return promise;
+  }
+
+  /**
+   * Gets a cached GET response. Returns `undefined` if missing or expired.
+   */
+  private cacheGet<T>(key: string): T | undefined {
+    const entry = this.cache.get(key);
+    if (!entry) return undefined;
+    if (Date.now() - entry.timestamp > CACHE_TTL) {
+      this.cache.delete(key);
+      return undefined;
+    }
+    return entry.data as T;
+  }
+
+  /**
+   * Stores a GET response in the cache with the current timestamp.
+   */
+  private cacheSet(key: string, data: unknown): void {
+    this.cache.set(key, { data, timestamp: Date.now() });
+  }
+
+  /**
+   * Builds the dedup key for a POST request.
+   *
+   * @param endpoint The endpoint path to post to
+   * @param body The optional request body
+   */
+  private cacheKey(endpoint: string, body?: Record<string, unknown>): string {
+    return body ? `${endpoint}:${JSON.stringify(body)}` : endpoint;
+  }
+
   /**
    * Makes a raw GET request to the llama-server.
    * This bypasses caching and deduplication.
@@ -107,15 +162,4 @@ export class ApiClient {
 
     return res.json();
   }
-
-  /**
-   * Sets a cache key
-   *
-   * @param endpoint The endpoint path to post to
-   * @param body The optional request body
-   * @returns The cache key
-   */
-  private cacheKey(endpoint: string, body?: Record<string, unknown>): string {
-    return body ? `${endpoint}:${JSON.stringify(body)}` : endpoint;
-  }
 }

package/src/constants.ts

@@ -3,6 +3,11 @@
  */
 export const PROVIDER_PREFIX = "llama-server";
 
+/**
+ * The settings key used in project/global settings.
+ */
+export const SETTINGS_KEY = "llamaSettings";
+
 /**
  * This provider's name
  */
@@ -58,6 +63,16 @@ export const REACT_TO_MODEL_SELECT = true;
  */
 export const AUTOLOAD_ON_MESSAGE = false;
 
+/**
+ * Default sort order for model lists.
+ */
+export const SORT_BY = "asc";
+
+/**
+ * Sort order options for model lists.
+ */
+export type SortBy = "asc" | "desc" | "asc-name" | "desc-name" | "api";
+
 /**
  * Thinking budgets to send to the server, depending on user-selected level in Pi.
  */

package/src/index.ts

@@ -14,11 +14,9 @@ import { ServerManager } from "./managers/server";
 import { settings } from "./managers/settings";
 
 export default async function (pi: ExtensionAPI) {
-  const servers = await settings.resolveServers();
-
-  const eventManager = new EventManager(servers);
-  const serverManager = new ServerManager(servers);
-  const commandManager = new CommandManager(serverManager);
+  const serverManager = new ServerManager(settings);
+  const eventManager = new EventManager(serverManager, settings);
+  const commandManager = new CommandManager(serverManager, settings);
 
   // Register providers once at startup
   await serverManager.initialize(pi);

package/src/interfaces/events.ts

@@ -1,3 +1,14 @@
-export interface ModelSelectEvent {
-  model: { id: string; provider: string };
-}
+import type { ExtensionEvent } from "@earendil-works/pi-coding-agent";
+
+/**
+ * pi-coding-agent does not re-export its `ModelSelectEvent` from the public
+ * API (it exists in the internal extensions/index but is omitted from the
+ * root re-export list, and the package's `exports` map blocks deep imports).
+ * It is therefore derived from the exported `ExtensionEvent` union, which
+ * yields pi's real event shape (`model: Model<any>`, `previousModel`,
+ * `source`) and tracks the API automatically.
+ */
+export type ModelSelectEvent = Extract<
+  ExtensionEvent,
+  { type: "model_select" }
+>;

package/src/interfaces/server.ts

@@ -0,0 +1,20 @@
+/**
+ * Identity of a llama.cpp server endpoint.
+ *
+ * Persisted counterpart: {@link LlamaServer} (`llamaSettings.servers`),
+ * whose `url`/`id`/`name` map onto these fields.
+ */
+export interface ServerOptions {
+  /**
+   * Base URL of the llama.cpp server (e.g. "http://127.0.0.1:8080").
+   */
+  baseUrl: string;
+  /**
+   * Custom provider ID; falls back to a URL-based one.
+   */
+  customId?: string;
+  /**
+   * Custom provider name suffix; falls back to the base URL.
+   */
+  customName?: string;
+}

package/src/interfaces/settings.ts

@@ -1,7 +1,9 @@
+import type { SortBy } from "../constants";
+
 /**
  * A description of a server in the "llamaSettings" key
  */
-interface LlamaServer {
+export interface LlamaServer {
   /**
    * The URL of the llama.cpp server.
    */
@@ -29,14 +31,16 @@ interface LlamaServer {
  *   }],
  *   "reactToModelSelect": true
  *   "autoloadOnMessage": false
+ *   "sortBy": "asc"
  * }
 }
  */
 export interface LlamaSettings {
   /**
    * List of servers to connect to.
+   * @default []
    */
-  servers: LlamaServer[];
+  servers?: LlamaServer[];
   /**
    * Whether to react to model selection events by loading the model.
    * @default true
@@ -57,4 +61,9 @@ export interface LlamaSettings {
    * @default 1000
    */
   serverTimeout?: number;
+  /**
+   * How to sort models in the /models command.
+   * @default "asc"
+   */
+  sortBy?: SortBy;
 }