pi-llama-cpp 0.7.0 → 0.7.2

package/README.md

@@ -127,10 +127,10 @@ llama-server --model path/to/model.gguf ...
 The extension determines the context size as follows:
 
 - **Router mode**
-  - When loaded, reads `meta.n_ctx` from the `/models` endpoint
+  - When loaded, reads `meta.n_ctx` from the `/v1/models` endpoint
   - When not loaded, reads `--ctx-size` and/or `--fit-ctx` from the server arguments (which can also originate from the **presets.ini** file the llama.cpp server uses to load its models).
-- **Single mode** — reads `meta.n_ctx` from the `/models` endpoint
-- **Legacy mode** — reads `max_model_len` from `/models`, falling back to `n_ctx` from `/props`
+- **Single mode** — reads `meta.n_ctx` from the `/v1/models` endpoint
+- **Legacy mode** — reads `max_model_len` from `/v1/models`, falling back to `n_ctx` from `/props`
 - Falls back to `128000` if not available
 
 ### Commands
@@ -209,7 +209,7 @@ When you trigger a load, switch, or retry action, the extension polls the server
 Each model exposed to Pi includes the following defaults:
 
 - **`maxTokens`** — dynamically set to the model's context window (detected from llama-server)
-- **`reasoning`** — `true` (assumed, as llama.cpp's `/models` endpoint does not expose it)
+- **`reasoning`** — `true` (assumed, as llama.cpp's `/v1/models` endpoint does not expose it)
 - **`cost`** — all zero (local models)
 
 ## Dependencies

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-llama-cpp",
-  "version": "0.7.0",
+  "version": "0.7.2",
   "description": "Pi extension for llama.cpp integration. Supports router, single and legacy models. Supports multiple servers.",
   "keywords": [
     "pi",
@@ -36,8 +36,8 @@
     "@earendil-works/pi-tui": "*"
   },
   "devDependencies": {
-    "@types/node": "^25.9.3",
+    "@types/node": "^26.0.0",
     "prettier-plugin-organize-imports": "^4.3.0",
-    "vitest": "^4.1.8"
+    "vitest": "^4.1.9"
   }
 }

package/src/managers/events.ts

@@ -10,7 +10,6 @@ import { Server } from "../server";
 
 export class EventManager {
   static inflightModel: BaseModel | null = null;
-  private readonly resolver = new ConfigResolver();
 
   constructor(private readonly servers: Server[]) {}
 
@@ -86,8 +85,9 @@ export class EventManager {
     if (!isLlamaCpp) return payload;
 
     // Retrieve pi's current thinking level, so we can setup a budget
-    const level = this.resolver.resolveThinkingLevel() ?? "medium";
-    const budgets = this.resolver.resolveThinkingBudgets();
+    const resolver = new ConfigResolver();
+    const level = resolver.resolveThinkingLevel() ?? "medium";
+    const budgets = resolver.resolveThinkingBudgets();
     const thinking_budget_tokens = budgets[level];
 
     // Setup payload

package/src/models/routerModel.ts

@@ -14,8 +14,7 @@ export class RouterModel extends BaseModel {
   }
 
   /**
-   * Workaround for the currently-bugged /models status detection
-   * (I suspect it was introduced in PR #22683 of llama.cpp)
+   * Workaround for /models status detection
    *
    * When a model is loaded for the very first time,
    * this workaround will try to poll to /props instead of /models

package/src/server.ts

@@ -1,4 +1,4 @@
-import { PROVIDER_NAME, PROVIDER_PREFIX } from "./constants";
+import { POLLING_INTERVAL, PROVIDER_NAME, PROVIDER_PREFIX } from "./constants";
 import { Mode } from "./enums/mode";
 import { ServerStatus } from "./enums/serverStatus";
 import { HealthEndpoint } from "./interfaces/endpoints/health";
@@ -9,10 +9,14 @@ import { LegacyModel } from "./models/legacyModel";
 import { RouterModel } from "./models/routerModel";
 import { SingleModel } from "./models/singleModel";
 import { ConfigResolver } from "./resolver";
+import { Cache } from "./utils/cache";
+import { Mutex } from "./utils/mutex";
 
 export class Server {
   public readonly models: BaseModel[] = [];
   private configResolver = new ConfigResolver();
+  private cache = new Cache(POLLING_INTERVAL / 2);
+  private mutex = new Mutex();
 
   constructor(readonly baseUrl: string) {}
 
@@ -39,9 +43,11 @@ export class Server {
   }
 
   /**
-   * Fetches models from the server and populates {@link models}
+   * Fetches models from the server and populates {@link models}.
+   * Clears the cache first so we always fetch fresh data.
    */
   async initialize() {
+    this.cache.clear();
     const { data } = await this.fetchModels();
     const mode = await this.detectServerMode();
 
@@ -150,11 +156,13 @@ export class Server {
     resource: "load" | "unload",
     model: string,
   ): Promise<ModelsEndpoint> {
+    this.cache.clear();
     return await this.rpc<ModelsEndpoint>(`/models/${resource}`, { model });
   }
 
   /**
-   * Makes an HTTP request to the llama-server and returns the parsed JSON response
+   * Makes a cached, deduplicated request to the llama-server.
+   * Results are cached for half the {@link POLLING_INTERVAL} and in-flight requests are deduplicated.
    *
    * @param endpoint The endpoint path to fetch (e.g. "/health")
    * @param body The optional request body for POST requests
@@ -163,6 +171,33 @@ export class Server {
   private async rpc<T>(
     endpoint: string,
     body?: Record<string, unknown>,
+  ): Promise<T> {
+    const key = this.cacheKey(endpoint, body);
+
+    // Check cache
+    const cached = this.cache.get<T>(key);
+    if (cached !== undefined) {
+      return cached;
+    }
+
+    // Deduplicate in-flight requests
+    return this.mutex.getOrCreate(key, async () => {
+      const data = await this.fetch<T>(endpoint, body);
+      this.cache.set(key, data);
+      return data;
+    });
+  }
+
+  /**
+   * Makes an HTTP request to the llama-server and returns the parsed JSON response.
+   *
+   * @param endpoint The endpoint path to fetch (e.g. "/health")
+   * @param body The optional request body for POST requests
+   * @returns The parsed JSON response from the server
+   */
+  private async fetch<T>(
+    endpoint: string,
+    body?: Record<string, unknown>,
   ): Promise<T> {
     const url = `${this.baseUrl}${endpoint}`;
     const apiKey = await this.getApiKey();
@@ -184,4 +219,15 @@ export class Server {
     const response: T = await res.json();
     return response;
   }
+
+  /**
+   * Generates a cache key from the endpoint and body.
+   *
+   * @param endpoint The endpoint path (e.g. "/health")
+   * @param body The optional request body for POST requests
+   * @returns A key used for caching
+   */
+  private cacheKey(endpoint: string, body?: Record<string, unknown>): string {
+    return body ? `${endpoint}:${JSON.stringify(body)}` : endpoint;
+  }
 }

package/src/utils/cache.ts

@@ -0,0 +1,39 @@
+/**
+ * Generic TTL cache.
+ * Entries expire after `ttl` milliseconds from the time they were set.
+ */
+export class Cache {
+  private entries = new Map<string, { data: unknown; timestamp: number }>();
+
+  /**
+   * @param ttl Time-to-live in milliseconds
+   */
+  constructor(private readonly ttl: number) {}
+
+  /**
+   * Gets a cached value by key. Returns `undefined` if missing or expired.
+   */
+  get<T>(key: string): T | undefined {
+    const entry = this.entries.get(key);
+    if (!entry) return undefined;
+    if (Date.now() - entry.timestamp > this.ttl) {
+      this.entries.delete(key);
+      return undefined;
+    }
+    return entry.data as T;
+  }
+
+  /**
+   * Stores a value in the cache with the current timestamp.
+   */
+  set(key: string, data: unknown): void {
+    this.entries.set(key, { data, timestamp: Date.now() });
+  }
+
+  /**
+   * Clears all cached entries.
+   */
+  clear(): void {
+    this.entries.clear();
+  }
+}

package/src/utils/mutex.ts

@@ -0,0 +1,24 @@
+/**
+ * Ensures only one in-flight operation exists per key.
+ * Concurrent callers for the same key share the same promise.
+ */
+export class Mutex {
+  private promises = new Map<string, Promise<unknown>>();
+
+  /**
+   * Runs `fn` for the given key, or returns an existing in-flight promise.
+   * Concurrent callers for the same key share the same promise.
+   */
+  getOrCreate<T>(key: string, fn: () => Promise<T>): Promise<T> {
+    const existing = this.promises.get(key);
+    if (existing) {
+      return existing as Promise<T>;
+    }
+
+    const promise = fn().finally(() => {
+      this.promises.delete(key);
+    });
+    this.promises.set(key, promise);
+    return promise;
+  }
+}