pi-llamacpp-infra 1.2.2 → 1.2.3

package/README.md

@@ -25,6 +25,7 @@ Anything else (vLLM, Ollama, cloud APIs…) is out of scope — use pi's built-i
 - **Multi-machine discovery** — configurable list of servers (host, ports, API key, options); probes all of them at startup and on demand
 - **Compact model ids** — models appear as `Name (host:port)` in pi's `/model` picker, like a native provider; the raw GGUF path/alias is sent to the server automatically on every request
 - **Single-model & router modes** — llama.cpp single-model mode (one GGUF per instance) and router mode (multiple models per server, with per-model status and args)
+- **Long local generations** — discovered models are registered with up to **32,768 output tokens** (bounded by the model/server context) and llamacpp-infra OpenAI-compatible requests enforce a **20 minute** timeout floor so slow local runs don't get cut early by pi defaults
 - **Per-model metadata badges** — 👁️ vision (mmproj / modalities), 🚀 drafter (speculative decoding), 🗜️ quant tag from GGUF filename, 🧠 KV cache quantization (from server args or `/proc`)
 - **Live speed & metrics** — a constantly updating footer reading of the active model's prefill (⚡) and generation (🔥) token speed, measured straight from the stream (per token, ~10 updates/s); when pi is idle it also mirrors other clients the server's `/metrics` endpoint reports. Lives in the footer's status line, so no extra terminal row is taken. Works even without `--metrics`
 - **Thinking budgets** — llama.cpp accepts `thinking_budget_tokens` per request; configure budgets per thinking level (minimal/low/medium/high/xhigh/max) per model; models with budgets are registered with reasoning enabled
@@ -223,6 +224,8 @@ Everything is configurable through the UI, but the persisted file is `~/.pi/agen
 | `prefixModelIds` | `true` | Append the machine tag `(host:port)` to model ids; OFF keeps bare names and only disambiguates collisions |
 | `showBadgesInNames` | `true` | Append 👁️🚀💤 badges to model display names |
 | `includeUnloadedRouterModels` | `false` | Router mode: list models that are not currently loaded |
+| output cap | `32768` | Registered per model as `maxTokens` unless the server reports an explicit `max_tokens`; still bounded by available context |
+| request timeout | `1200000` | 20 minute timeout floor applied to llamacpp-infra OpenAI-compatible streams |
 | `warmup` | `true` | Pre-cache system prompt KV on llama.cpp servers |
 | `metricsEnabled` | `true` | Show live speed & metrics in the footer for llamacpp-infra models |
 | `metricsPollMs` | `5000` | How often `/metrics` is fetched |

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "pi-llamacpp-infra",
-  "version": "1.2.2",
+  "version": "1.2.3",
   "description": "Discovery, metrics and control of llama.cpp-family servers for pi (llama.cpp, ZINC, DwarfStar/ds4, lucebox, LM Studio): scan machines (localhost, LAN, Tailscale), register models, live metrics, per-model thinking budgets, vision detection, native config UI.",
   "keywords": [
     "pi-package",

package/src/core.ts

@@ -21,6 +21,8 @@ export const LEGACY_CONFIG_FILE = "local-models.json";
 export const METRICS_STATUS_KEY = "llamacpp-infra-speed";
 export const DEFAULT_API_KEY = "no-auth";
 export const THINKING_BUDGET_FIELD = "thinking_budget_tokens";
+export const DEFAULT_MAX_OUTPUT_TOKENS = 32_768;
+export const DEFAULT_PROVIDER_TIMEOUT_MS = 20 * 60 * 1000;
 
 // ── Settings defaults ───────────────────────────────────────────────────────
 export const DEFAULT_SETTINGS: SettingsConfig = {
@@ -28,6 +30,8 @@ export const DEFAULT_SETTINGS: SettingsConfig = {
 	pollIntervalMs: 4000,
 	pollMaxMs: 90_000,
 	startupGraceMs: 40_000,
+	maxOutputTokens: DEFAULT_MAX_OUTPUT_TOKENS,
+	requestTimeoutMs: DEFAULT_PROVIDER_TIMEOUT_MS,
 	knownGoodFailLimit: 3,
 	detectVision: true,
 	prefixModelIds: true,

package/src/index.ts

@@ -20,6 +20,7 @@ import {
 	shared,
 	supportsThinkingBudget,
 } from "./core.ts";
+import { createLongTimeoutOpenAICompletionsStream } from "./runtime.ts";
 
 export default function (pi: ExtensionAPI) {
 	const config = loadConfig();
@@ -120,6 +121,7 @@ export default function (pi: ExtensionAPI) {
 			baseUrl: `http://${first?.host ?? "127.0.0.1"}:${first?.ports[0] ?? 8080}/v1`,
 			apiKey: first?.apiKey || "no-auth",
 			api: "openai-completions",
+			streamSimple: createLongTimeoutOpenAICompletionsStream,
 			models: [],
 		});
 		providerIsEmpty = true;

package/src/registration.ts

@@ -21,6 +21,7 @@ import type {
 	ScanResult,
 } from "./types.ts";
 import { cleanModelName, lmStudioContextLength } from "./scan.ts";
+import { createLongTimeoutOpenAICompletionsStream } from "./runtime.ts";
 
 /** Compact badge string appended to display names when enabled. */
 function badgeSuffix(
@@ -73,7 +74,7 @@ function toPiModel(
 			name: `${displayName}${badgeSuffix(modelMeta, settings.showBadgesInNames)}`,
 			reasoning: false,
 			contextWindow,
-			maxTokens: model.max_tokens ?? Math.min(contextWindow, 8192),
+			maxTokens: model.max_tokens ?? Math.min(contextWindow, settings.maxOutputTokens),
 			compat: makeCompat(kind),
 		};
 	}
@@ -92,7 +93,7 @@ function toPiModel(
 			name: `${displayName}${badgeSuffix(modelMeta, settings.showBadgesInNames)}`,
 			reasoning: ep.props?.capabilities?.reasoning_supported ?? true,
 			contextWindow,
-			maxTokens: model.max_tokens ?? Math.min(contextWindow, 8192),
+			maxTokens: model.max_tokens ?? Math.min(contextWindow, settings.maxOutputTokens),
 			compat: makeCompat(kind),
 		};
 	}
@@ -118,7 +119,7 @@ function toPiModel(
 		name: `${displayName}${badgeSuffix(modelMeta, settings.showBadgesInNames)}`,
 		reasoning: isLlamaFamily,
 		contextWindow,
-		maxTokens: model.max_tokens ?? Math.min(contextWindow, 8192),
+		maxTokens: model.max_tokens ?? Math.min(contextWindow, settings.maxOutputTokens),
 		compat: makeCompat(kind),
 	};
 }
@@ -232,6 +233,7 @@ export function buildAndRegisterProvider(pi: ExtensionAPI, scan: ScanResult, con
 		baseUrl: defaultBaseUrl,
 		apiKey: defaultApiKey,
 		api: "openai-completions",
+		streamSimple: createLongTimeoutOpenAICompletionsStream,
 		models: piModels,
 	});
 

package/src/runtime.ts

@@ -0,0 +1,131 @@
+// Runtime request defaults for the OpenAI-compatible llama.cpp provider.
+// pi owns the normal SDK stream options, but this provider is for slow local
+// models where long generations are expected. Keep a local floor so llamacpp-
+// infra models don't inherit too-small global/default request timeouts.
+
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { DEFAULT_PROVIDER_TIMEOUT_MS, shared } from "./core.ts";
+
+type AssistantMessageEvent = any;
+type AssistantMessage = any;
+type StreamOptions = Record<string, any> | undefined;
+type StreamSimple = (model: any, context: any, options?: Record<string, any>) => AsyncIterable<AssistantMessageEvent> & { result?: () => Promise<AssistantMessage> };
+
+class ForwardedAssistantMessageEventStream implements AsyncIterable<AssistantMessageEvent> {
+	private queue: AssistantMessageEvent[] = [];
+	private waiting: Array<(result: IteratorResult<AssistantMessageEvent>) => void> = [];
+	private done = false;
+	private resolveFinal!: (value: AssistantMessage) => void;
+	private readonly finalResult = new Promise<AssistantMessage>((resolve) => {
+		this.resolveFinal = resolve;
+	});
+
+	push(event: AssistantMessageEvent): void {
+		if (this.done) return;
+		if (event?.type === "done") {
+			this.done = true;
+			this.resolveFinal(event.message);
+		} else if (event?.type === "error") {
+			this.done = true;
+			this.resolveFinal(event.error);
+		}
+
+		const waiter = this.waiting.shift();
+		if (waiter) waiter({ value: event, done: false });
+		else this.queue.push(event);
+	}
+
+	end(result?: AssistantMessage): void {
+		this.done = true;
+		if (result !== undefined) this.resolveFinal(result);
+		while (this.waiting.length > 0) this.waiting.shift()?.({ value: undefined, done: true });
+	}
+
+	async *[Symbol.asyncIterator](): AsyncIterator<AssistantMessageEvent> {
+		for (;;) {
+			if (this.queue.length > 0) {
+				yield this.queue.shift();
+			} else if (this.done) {
+				return;
+			} else {
+				const next = await new Promise<IteratorResult<AssistantMessageEvent>>((resolve) => this.waiting.push(resolve));
+				if (next.done) return;
+				yield next.value;
+			}
+		}
+	}
+
+	result(): Promise<AssistantMessage> {
+		return this.finalResult;
+	}
+}
+
+let openAICompletionsStreamPromise: Promise<StreamSimple> | undefined;
+
+async function loadOpenAICompletionsStreamSimple(): Promise<StreamSimple> {
+	if (!openAICompletionsStreamPromise) {
+		openAICompletionsStreamPromise = (async () => {
+			try {
+				const mod = await import("@earendil-works/pi-ai/api/openai-completions");
+				return (mod as { streamSimple: StreamSimple }).streamSimple;
+			} catch {
+				// In pi package installs, pi-ai may be nested under pi-coding-agent
+				// instead of hoisted as a top-level dependency of this extension.
+				const piIndexUrl = import.meta.resolve("@earendil-works/pi-coding-agent");
+				const piPackageDir = dirname(dirname(fileURLToPath(piIndexUrl)));
+				const nestedModule = join(piPackageDir, "node_modules", "@earendil-works", "pi-ai", "dist", "api", "openai-completions.js");
+				const mod = await import(pathToFileURL(nestedModule).href);
+				return (mod as { streamSimple: StreamSimple }).streamSimple;
+			}
+		})();
+	}
+	return openAICompletionsStreamPromise;
+}
+
+export function activeRequestTimeoutMs(): number {
+	const configured = shared.activeConfig?.settings?.requestTimeoutMs;
+	return typeof configured === "number" && Number.isFinite(configured) && configured > 0
+		? configured
+		: DEFAULT_PROVIDER_TIMEOUT_MS;
+}
+
+export function withLocalRuntimeDefaults(options: StreamOptions, floorMs?: number): Record<string, any> {
+	const floor = typeof floorMs === "number" && Number.isFinite(floorMs) && floorMs > 0 ? floorMs : DEFAULT_PROVIDER_TIMEOUT_MS;
+	const currentTimeout = typeof options?.timeoutMs === "number" && Number.isFinite(options.timeoutMs) ? options.timeoutMs : undefined;
+	return {
+		...(options ?? {}),
+		timeoutMs: currentTimeout === undefined ? floor : Math.max(currentTimeout, floor),
+	};
+}
+
+export function createLongTimeoutOpenAICompletionsStream(model: any, context: any, options?: Record<string, any>) {
+	const out = new ForwardedAssistantMessageEventStream();
+	void (async () => {
+		try {
+			const streamSimple = await loadOpenAICompletionsStreamSimple();
+			const inner = streamSimple(model, context, withLocalRuntimeDefaults(options, activeRequestTimeoutMs()));
+			for await (const event of inner) out.push(event);
+			if (typeof inner.result === "function") out.end(await inner.result());
+			else out.end();
+		} catch (err) {
+			const message = err instanceof Error ? err.message : String(err);
+			out.push({
+				type: "error",
+				reason: "error",
+				error: {
+					role: "assistant",
+					content: [],
+					api: model?.api ?? "openai-completions",
+					provider: model?.provider ?? "llamacpp-infra",
+					model: model?.id ?? "unknown",
+					usage: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, totalTokens: 0, cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0, total: 0 } },
+					stopReason: "error",
+					errorMessage: message,
+					timestamp: Date.now(),
+				},
+			});
+		}
+	})();
+	return out;
+}

package/src/types.ts

@@ -31,6 +31,8 @@ export interface SettingsConfig {
 	pollIntervalMs: number;
 	pollMaxMs: number;
 	startupGraceMs: number;
+	maxOutputTokens: number;
+	requestTimeoutMs: number;
 	knownGoodFailLimit: number;
 	detectVision: boolean;
 	prefixModelIds: boolean;

package/src/ui.ts

@@ -64,6 +64,15 @@ function formatCtx(tokens: number | undefined): string {
 	return tokens >= 1024 ? `${Math.round(tokens / 1024)}k` : `${tokens}`;
 }
 
+function formatTokens(n: number): string {
+	if (!Number.isFinite(n) || n <= 0) return "0";
+	if (n >= 1024) {
+		const k = n / 1024;
+		return Number.isInteger(k) ? `${k}k` : `${k.toFixed(1)}k`;
+	}
+	return `${n}`;
+}
+
 function metadataBadges(m: ModelMetadata | undefined): string {
 	if (!m) return "";
 	const parts: string[] = [];
@@ -661,6 +670,16 @@ async function showSettingsMenu(ctx: ExtensionContext, deps: UiDeps): Promise<vo
 				label: `💤 Include unloaded router models: ${s.includeUnloadedRouterModels ? "ON" : "OFF"}`,
 				description: "router mode: list models not currently loaded",
 			},
+			{
+				value: "maxtokens",
+				label: `📏 Max output tokens: ${formatTokens(s.maxOutputTokens)}`,
+				description: "per-model ceiling for outgoing max_tokens requests",
+			},
+			{
+				value: "timeoutfloor",
+				label: `⏱️ Request timeout: ${formatMs(s.requestTimeoutMs)}`,
+				description: "lower bound on OpenAI-completions stream idle timeout",
+			},
 			{
 				value: "warmup",
 				label: `☕ Header warmup: ${s.warmup ? "ON" : "OFF"}`,
@@ -765,6 +784,25 @@ async function showSettingsMenu(ctx: ExtensionContext, deps: UiDeps): Promise<vo
 				saveConfig(config);
 				ctx.ui.notify(`☕ Header warmup ${config.settings.warmup ? "ON" : "OFF"}`, "info");
 				break;
+			case "maxtokens": {
+				const v = await pickNumber("📏 Max output tokens", [4_096, 8_192, 16_384, 24_576, 32_768, 49_152, 65_536]);
+				if (v !== undefined) {
+					config.settings.maxOutputTokens = v;
+					saveConfig(config);
+					ctx.ui.notify(`📏 Max output tokens: ${v.toLocaleString()}`, "info");
+					await deps.rescan(ctx);
+				}
+				break;
+			}
+			case "timeoutfloor": {
+				const v = await pickNumber("⏱️ Request timeout", [60_000, 300_000, 600_000, 1_200_000, 1_800_000, 3_600_000]);
+				if (v !== undefined) {
+					config.settings.requestTimeoutMs = v;
+					saveConfig(config);
+					ctx.ui.notify(`⏱️ Request timeout: ${formatMs(v)}`, "info");
+				}
+				break;
+			}
 			case "reset": {
 				const ok = await ctx.ui.confirm("♻️ Reset settings", "Restore all discovery settings to their defaults?");
 				if (ok) {