peerllm-host-cli 1.9.1 → 1.9.4

package/dist/core/shared-runner.js

@@ -1,14 +1,24 @@
+import * as cp from "node:child_process";
 import { existsSync, promises as fsp } from "node:fs";
 import { arch as osArch, platform as osPlatform } from "node:os";
 import { join } from "node:path";
-import { getLlama, LlamaChatSession } from "node-llama-cpp";
+import { fileURLToPath } from "node:url";
 import si from "systeminformation";
 import { getLogger } from "./logger.js";
 import { calculateGpuLayersFromConfig, parseModelFilename, } from "./model-info.js";
 import { getLLMDirectory } from "./models-fs.js";
+// ---------------------------------------------------------------------------
+// Module-level constants
+// ---------------------------------------------------------------------------
 const DEFAULT_IDLE_TIMEOUT_MS = 10 * 60 * 1000;
 const WATCHDOG_INTERVAL_MS = 60 * 1000;
 const STALE_CONTEXT_LIMIT_MS = 5 * 60 * 1000;
+const WORKER_READY_TIMEOUT_MS = 10_000;
+const DISPOSE_SAFETY_TIMEOUT_MS = 5_000;
+const workerScript = fileURLToPath(new URL("./llmWorker.js", import.meta.url));
+// ---------------------------------------------------------------------------
+// Graphics cache (used for GPU layer calculation in parent process)
+// ---------------------------------------------------------------------------
 let _cachedGraphics;
 let _graphicsInFlight;
 const GRAPHICS_CACHE_TTL_MS = 5 * 60 * 1000;
@@ -26,15 +36,28 @@ async function getCachedGraphics() {
     }
     return _graphicsInFlight;
 }
+// ---------------------------------------------------------------------------
+// SharedGGUFRunner
+// ---------------------------------------------------------------------------
 export class SharedGGUFRunner {
     env;
     idleTimeoutMs;
-    llama = null;
-    model = null;
+    // Worker process
+    worker = null;
+    _workerEra = 0;
+    _gracefulDispose = false;
+    _workerCrashCallback = null;
+    // Lightweight context trackers — actual objects live in the worker
+    contextTrackers = new Map();
+    maxConcurrentContexts = 10;
+    // IPC promise bridges
+    _loadCallback = null;
+    _contextCallbacks = new Map();
+    _promptCallbacks = new Map();
+    _disposeCallback = null;
+    // Runner state
     modelName = "";
     gpuLayers = 0;
-    contexts = new Map();
-    maxConcurrentContexts = 10;
     runnerState = "cold";
     modelState = null;
     activeRequestCount = 0;
@@ -50,6 +73,9 @@ export class SharedGGUFRunner {
     setIdleDisposeCallback(cb) {
         this.onIdleDispose = cb;
     }
+    setWorkerCrashCallback(cb) {
+        this._workerCrashCallback = cb;
+    }
     getModelState() {
         return this.modelState;
     }
@@ -61,7 +87,7 @@ export class SharedGGUFRunner {
     }
     getStats() {
         return {
-            activeContexts: this.contexts.size,
+            activeContexts: this.contextTrackers.size,
             maxContexts: this.maxConcurrentContexts,
             modelName: this.modelName,
             gpuLayers: this.gpuLayers,
@@ -71,7 +97,7 @@ export class SharedGGUFRunner {
         if (this.runnerState === "disposing") {
             throw new Error(`Cannot load "${modelName}" while disposal is in progress`);
         }
-        if (this.model && this.modelName === modelName) {
+        if (this.runnerState === "loaded" && this.modelName === modelName) {
             if (this.modelState && loadType === "manual" && this.modelState.loadType === "auto") {
                 this.modelState.loadType = "manual";
                 this.clearIdleTimer();
@@ -97,7 +123,6 @@ export class SharedGGUFRunner {
             model: modelName,
         });
         try {
-            this.llama = await getLlama();
             const llmDir = getLLMDirectory(this.env.paths, this.env.config);
             const modelPath = join(llmDir, `${modelName}.gguf`);
             const graphics = await getCachedGraphics();
@@ -130,7 +155,7 @@ export class SharedGGUFRunner {
                 this._cleanupPartialLoad();
                 return;
             }
-            this.model = await this.llama.loadModel({ modelPath, gpuLayers: this.gpuLayers });
+            await this._spawnAndLoad(modelPath, this.gpuLayers);
             if (this.generation !== gen) {
                 this._cleanupPartialLoad();
                 return;
@@ -157,79 +182,288 @@ export class SharedGGUFRunner {
             throw new Error(`Failed to load '${modelName}': ${message}`);
         }
     }
+    // ---------------------------------------------------------------------------
+    // Worker lifecycle
+    // ---------------------------------------------------------------------------
+    async _spawnAndLoad(modelPath, gpuLayers) {
+        const era = ++this._workerEra;
+        const worker = cp.fork(workerScript, [], {
+            stdio: ["ignore", "ignore", "inherit"],
+            serialization: "json",
+        });
+        this.worker = worker;
+        // Self-invalidating exit handler: ignores stale exits after era increments
+        worker.on("exit", (code, signal) => {
+            if (this._workerEra !== era)
+                return;
+            this._handleWorkerExit(code, signal);
+        });
+        // Await the initial 'ready' handshake, then switch to permanent handler
+        await new Promise((resolve, reject) => {
+            const t = setTimeout(() => {
+                reject(new Error("llmWorker: ready timeout"));
+            }, WORKER_READY_TIMEOUT_MS);
+            const onMsg = (raw) => {
+                const msg = raw;
+                if (msg?.type === "ready") {
+                    clearTimeout(t);
+                    worker.off("message", onMsg);
+                    worker.on("message", (m) => this._handleWorkerMessage(m));
+                    resolve();
+                }
+            };
+            worker.on("message", onMsg);
+            worker.once("error", (err) => {
+                clearTimeout(t);
+                reject(err);
+            });
+        });
+        // Send load-model and await confirmation
+        await new Promise((resolve, reject) => {
+            this._loadCallback = { resolve, reject, gen: this.generation };
+            this._workerSend({ type: "load-model", modelPath, gpuLayers });
+        });
+        this._loadCallback = null;
+    }
+    _handleWorkerMessage(msg) {
+        switch (msg.type) {
+            case "loaded": {
+                const cb = this._loadCallback;
+                this._loadCallback = null;
+                cb?.resolve();
+                break;
+            }
+            case "context-created": {
+                this.contextTrackers.set(msg.conversationId, {
+                    lastUsedAt: Date.now(),
+                    inFlight: false,
+                });
+                const cb = this._contextCallbacks.get(msg.conversationId);
+                if (cb) {
+                    this._contextCallbacks.delete(msg.conversationId);
+                    cb.resolve();
+                }
+                break;
+            }
+            case "token": {
+                const cb = this._promptCallbacks.get(msg.requestId);
+                if (cb && this.generation === cb.gen) {
+                    cb.onToken?.(msg.chunk);
+                }
+                break;
+            }
+            case "done": {
+                const cb = this._promptCallbacks.get(msg.requestId);
+                if (cb) {
+                    this._promptCallbacks.delete(msg.requestId);
+                    cb.resolve({ text: msg.text, promptTokens: msg.promptTokens, completionTokens: msg.completionTokens });
+                }
+                break;
+            }
+            case "cancelled": {
+                const cb = this._promptCallbacks.get(msg.requestId);
+                if (cb) {
+                    this._promptCallbacks.delete(msg.requestId);
+                    cb.resolve({ text: msg.text, promptTokens: msg.promptTokens, completionTokens: msg.completionTokens });
+                }
+                break;
+            }
+            case "error": {
+                if (msg.requestId) {
+                    const cb = this._promptCallbacks.get(msg.requestId);
+                    if (cb) {
+                        this._promptCallbacks.delete(msg.requestId);
+                        cb.reject(new Error(msg.message));
+                    }
+                }
+                else if (msg.conversationId) {
+                    const cb = this._contextCallbacks.get(msg.conversationId);
+                    if (cb) {
+                        this._contextCallbacks.delete(msg.conversationId);
+                        cb.reject(new Error(msg.message));
+                    }
+                }
+                else {
+                    // Load error
+                    const cb = this._loadCallback;
+                    this._loadCallback = null;
+                    cb?.reject(new Error(msg.message));
+                }
+                break;
+            }
+            case "disposed": {
+                const cb = this._disposeCallback;
+                this._disposeCallback = null;
+                cb?.resolve();
+                this._finalizeDispose();
+                break;
+            }
+            case "log": {
+                const level = msg.level;
+                getLogger()[level]?.(`[llmWorker] ${msg.message}`);
+                break;
+            }
+            default:
+                break;
+        }
+    }
+    _handleWorkerExit(code, signal) {
+        if (this._gracefulDispose) {
+            // Expected shutdown — resolve dispose callback if not already done by 'disposed' message
+            const cb = this._disposeCallback;
+            this._disposeCallback = null;
+            cb?.resolve();
+            return;
+        }
+        // Unexpected crash
+        const crashModel = this.modelName;
+        const timestamp = Date.now();
+        getLogger().error(`llmWorker crashed: model="${crashModel}" code=${code ?? "null"} signal=${signal ?? "null"}`);
+        // Reject all pending load/context/prompt callbacks
+        const loadCb = this._loadCallback;
+        this._loadCallback = null;
+        loadCb?.reject(new Error(`llmWorker crashed during load (code=${String(code)})`));
+        for (const [, cb] of this._contextCallbacks.entries()) {
+            cb.reject(new Error(`llmWorker crashed (code=${String(code)})`));
+        }
+        this._contextCallbacks.clear();
+        for (const [, cb] of this._promptCallbacks.entries()) {
+            cb.reject(new Error(`llmWorker crashed (code=${String(code)})`));
+        }
+        this._promptCallbacks.clear();
+        const crashCb = this._workerCrashCallback;
+        this._finalizeDispose();
+        crashCb?.({ model: crashModel, timestamp });
+        this.onIdleDispose?.(crashModel);
+    }
+    _finalizeDispose() {
+        this.worker = null;
+        this.modelName = "";
+        this.modelState = null;
+        this.activeRequestCount = 0;
+        this.contextTrackers.clear();
+        if (this._loadCallback) {
+            this._loadCallback.reject(new Error("runner disposed"));
+            this._loadCallback = null;
+        }
+        for (const [, cb] of this._contextCallbacks.entries()) {
+            cb.reject(new Error("runner disposed"));
+        }
+        this._contextCallbacks.clear();
+        for (const [, cb] of this._promptCallbacks.entries()) {
+            cb.reject(new Error("runner disposed"));
+        }
+        this._promptCallbacks.clear();
+        if (this._disposeCallback) {
+            this._disposeCallback.resolve();
+            this._disposeCallback = null;
+        }
+        this.clearIdleTimer();
+        this.stopWatchdog();
+        this.runnerState = "cold";
+    }
+    _workerSend(msg) {
+        if (!this.worker)
+            return;
+        try {
+            this.worker.send(msg);
+        }
+        catch {
+            // Worker may have already exited
+        }
+    }
     _cleanupPartialLoad() {
+        if (this.runnerState === "cold")
+            return; // already finalized by exit handler
         this.clearIdleTimer();
         this.stopWatchdog();
-        this.safeDispose(this.model, "model (partial)");
-        this.safeDispose(this.llama, "llama (partial)");
-        for (const [id, entry] of this.contexts.entries()) {
-            this.safeDispose(entry.context, `context[${id}] (partial)`);
-        }
-        this.contexts.clear();
-        this.model = null;
-        this.llama = null;
+        ++this._workerEra; // invalidate exit handler for this era
+        if (this.worker) {
+            try {
+                this.worker.kill();
+            }
+            catch {
+                // ignored
+            }
+            this.worker = null;
+        }
+        this.contextTrackers.clear();
         this.modelName = "";
         this.modelState = null;
         this.activeRequestCount = 0;
         this.runnerState = "cold";
     }
+    // ---------------------------------------------------------------------------
+    // Context management
+    // ---------------------------------------------------------------------------
+    async _ensureContext(conversationId, gen) {
+        const existing = this.contextTrackers.get(conversationId);
+        if (existing) {
+            existing.lastUsedAt = Date.now();
+            return;
+        }
+        // LRU eviction if at capacity
+        if (this.contextTrackers.size >= this.maxConcurrentContexts) {
+            const evictable = Array.from(this.contextTrackers.entries())
+                .filter(([, t]) => !t.inFlight)
+                .sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
+            if (evictable.length > 0 && evictable[0]) {
+                const [evictId] = evictable[0];
+                this.contextTrackers.delete(evictId);
+                this._workerSend({ type: "clear-context", conversationId: evictId });
+            }
+        }
+        await new Promise((resolve, reject) => {
+            this._contextCallbacks.set(conversationId, { resolve, reject, gen });
+            this._workerSend({ type: "create-context", conversationId });
+        });
+        // Tracker entry is set by _handleWorkerMessage when 'context-created' arrives
+    }
+    // ---------------------------------------------------------------------------
+    // Public inference API
+    // ---------------------------------------------------------------------------
     async prompt(conversationId, input, onToken, options) {
-        if (!this.model)
+        if (this.runnerState !== "loaded")
             throw new Error("Model not loaded.");
         const gen = this.generation;
         this.activeRequestCount++;
         this.clearIdleTimer();
         if (this.modelState)
             this.modelState.lastUsedAt = Date.now();
-        const promptTokens = this.model.tokenize(input).length;
-        const { session } = await this.getOrCreateContext(conversationId);
-        const entry = this.contexts.get(conversationId);
-        if (!entry) {
+        await this._ensureContext(conversationId, gen);
+        const tracker = this.contextTrackers.get(conversationId);
+        if (!tracker) {
             this.activeRequestCount = Math.max(0, this.activeRequestCount - 1);
             throw new Error(`Context for "${conversationId}" was evicted before prompt could start`);
         }
-        entry.abortController = new AbortController();
-        entry.inFlight = true;
-        let fullText = "";
+        tracker.inFlight = true;
+        const requestId = `${conversationId}:${Date.now()}:${Math.random().toString(36).slice(2)}`;
         try {
-            const promptOptions = {
-                onTextChunk: (chunk) => {
-                    if (this.generation !== gen)
-                        return;
-                    if (onToken)
-                        onToken(chunk);
-                    fullText += chunk;
-                },
-                signal: entry.abortController.signal,
-            };
-            if (options?.maxTokens !== undefined)
-                promptOptions["maxTokens"] = options.maxTokens;
-            if (options?.temperature !== undefined)
-                promptOptions["temperature"] = options.temperature;
-            if (options?.stop?.length)
-                promptOptions["customStopTriggers"] = options.stop;
-            await session.prompt(input, promptOptions);
+            const result = await new Promise((resolve, reject) => {
+                this._promptCallbacks.set(requestId, {
+                    resolve,
+                    reject,
+                    onToken,
+                    gen,
+                    conversationId,
+                });
+                this._workerSend({ type: "prompt", requestId, conversationId, input, options });
+            });
+            this._promptCallbacks.delete(requestId);
             if (this.generation !== gen) {
-                return { text: fullText.trim(), promptTokens, completionTokens: 0 };
+                return { text: result.text.trim(), promptTokens: result.promptTokens, completionTokens: 0 };
             }
-            fullText = fullText.trim();
-            const completionTokens = this.model.tokenize(fullText).length;
-            return { text: fullText, promptTokens, completionTokens };
+            return result;
         }
         catch (err) {
-            const e = err;
-            if (e?.name === "AbortError") {
-                const completionTokens = this.generation === gen && this.model ? this.model.tokenize(fullText).length : 0;
-                return { text: fullText, promptTokens, completionTokens };
-            }
+            this._promptCallbacks.delete(requestId);
             throw err;
         }
         finally {
             if (this.generation === gen) {
-                if (entry) {
-                    entry.abortController = null;
-                    entry.inFlight = false;
-                    entry.lastUsedAt = Date.now();
+                if (tracker) {
+                    tracker.inFlight = false;
+                    tracker.lastUsedAt = Date.now();
                 }
                 this.activeRequestCount = Math.max(0, this.activeRequestCount - 1);
                 this.scheduleIdleCheck();
@@ -237,41 +471,15 @@ export class SharedGGUFRunner {
         }
     }
     cancel(conversationId) {
-        const entry = this.contexts.get(conversationId);
-        if (entry?.abortController) {
-            try {
-                entry.abortController.abort();
-            }
-            catch {
-                // ignored
-            }
-        }
+        this._workerSend({ type: "cancel", conversationId });
     }
     cancelAll() {
-        for (const entry of this.contexts.values()) {
-            if (entry.abortController) {
-                try {
-                    entry.abortController.abort();
-                }
-                catch {
-                    // ignored
-                }
-            }
-        }
+        this._workerSend({ type: "cancel-all" });
     }
     clearConversation(conversationId) {
-        const entry = this.contexts.get(conversationId);
-        if (entry) {
-            if (entry.inFlight) {
-                try {
-                    entry.abortController?.abort();
-                }
-                catch {
-                    // ignored
-                }
-            }
-            this.contexts.delete(conversationId);
-            this.safeDispose(entry.context, `context[${conversationId}]`);
+        if (this.contextTrackers.has(conversationId)) {
+            this.contextTrackers.delete(conversationId);
+            this._workerSend({ type: "clear-context", conversationId });
         }
     }
     async dispose(reason = "manual-dispose") {
@@ -287,10 +495,8 @@ export class SharedGGUFRunner {
                 // load failed; we're cold now
             }
         }
-        const stateAfter = this.runnerState;
-        if (stateAfter === "cold")
-            return;
-        if (stateAfter === "disposing")
+        const stateCheck = this.runnerState;
+        if (stateCheck === "cold" || stateCheck === "disposing")
             return;
         const prev = this.runnerState;
         this.runnerState = "disposing";
@@ -298,27 +504,42 @@ export class SharedGGUFRunner {
         this.logTransition(prev, "disposing", reason);
         this.clearIdleTimer();
         this.stopWatchdog();
-        this.cancelAll();
-        await new Promise((resolve) => setTimeout(resolve, 300));
-        const entries = Array.from(this.contexts.entries());
-        this.contexts.clear();
-        for (const [id, entry] of entries) {
-            this.safeDispose(entry.context, `context[${id}]`);
-        }
-        await new Promise((resolve) => setTimeout(resolve, 100));
-        this.safeDispose(this.model, `model[${this.modelName}]`);
-        this.safeDispose(this.llama, `llama[${this.modelName}]`);
-        this.model = null;
-        this.llama = null;
-        this.modelName = "";
-        this.modelState = null;
-        this.activeRequestCount = 0;
-        this.runnerState = "cold";
+        this._gracefulDispose = true;
+        const disposePromise = new Promise((resolve, reject) => {
+            this._disposeCallback = { resolve, reject };
+        });
+        this._workerSend({ type: "dispose" });
+        const safetyTimer = setTimeout(() => {
+            ++this._workerEra;
+            if (this.worker) {
+                try {
+                    this.worker.kill();
+                }
+                catch {
+                    // ignored
+                }
+            }
+            this._disposeCallback?.resolve();
+            this._disposeCallback = null;
+        }, DISPOSE_SAFETY_TIMEOUT_MS);
+        try {
+            await disposePromise;
+        }
+        finally {
+            clearTimeout(safetyTimer);
+            this._gracefulDispose = false;
+        }
+        const stateAfterDispose = this.runnerState;
+        if (stateAfterDispose !== "cold")
+            this._finalizeDispose();
         this.logTransition("disposing", "cold", reason);
     }
+    // ---------------------------------------------------------------------------
+    // Idle / watchdog timers
+    // ---------------------------------------------------------------------------
     scheduleIdleCheck() {
         this.clearIdleTimer();
-        if (!this.model)
+        if (this.runnerState !== "loaded")
             return;
         if (this.activeRequestCount > 0)
             return;
@@ -331,7 +552,7 @@ export class SharedGGUFRunner {
                 return;
             if (this.activeRequestCount > 0)
                 return;
-            if (!this.model)
+            if (this.runnerState !== "loaded")
                 return;
             const nameBefore = this.modelName;
             try {
@@ -360,57 +581,31 @@ export class SharedGGUFRunner {
         }
     }
     runWatchdogCheck() {
-        if (!this.model) {
+        if (this.runnerState !== "loaded") {
             this.stopWatchdog();
             return;
         }
         const checkGen = this.generation;
         const now = Date.now();
-        for (const [id, entry] of this.contexts.entries()) {
+        for (const [id, tracker] of this.contextTrackers.entries()) {
             if (this.generation !== checkGen)
                 break;
-            if (entry.inFlight)
+            if (tracker.inFlight)
                 continue;
-            const idleMs = now - entry.lastUsedAt;
+            const idleMs = now - tracker.lastUsedAt;
             if (idleMs <= STALE_CONTEXT_LIMIT_MS)
                 continue;
-            this.contexts.delete(id);
+            this.contextTrackers.delete(id);
+            this._workerSend({ type: "clear-context", conversationId: id });
             this.logTransition(this.runnerState, this.runnerState, "watchdog-evict", {
                 contextId: id,
                 idleMin: Math.floor(idleMs / 60000),
             });
-            this.safeDispose(entry.context, `watchdog-evict-context[${id}]`);
-        }
-    }
-    async getOrCreateContext(conversationId) {
-        const existing = this.contexts.get(conversationId);
-        if (existing) {
-            existing.lastUsedAt = Date.now();
-            return { context: existing.context, session: existing.session };
-        }
-        if (this.contexts.size >= this.maxConcurrentContexts) {
-            const evictable = Array.from(this.contexts.entries())
-                .filter(([, e]) => !e.inFlight)
-                .sort((a, b) => a[1].lastUsedAt - b[1].lastUsedAt);
-            if (evictable.length > 0 && evictable[0]) {
-                const [id, entry] = evictable[0];
-                this.contexts.delete(id);
-                this.safeDispose(entry.context, `lru-context[${id}]`);
-            }
         }
-        if (!this.model)
-            throw new Error("Model not loaded when creating context");
-        const context = await this.model.createContext({ contextSize: 4096 });
-        const session = new LlamaChatSession({ contextSequence: context.getSequence() });
-        this.contexts.set(conversationId, {
-            context,
-            session,
-            lastUsedAt: Date.now(),
-            inFlight: false,
-            abortController: null,
-        });
-        return { context, session };
     }
+    // ---------------------------------------------------------------------------
+    // Helpers
+    // ---------------------------------------------------------------------------
     async validateModelFile(modelPath, modelName) {
         if (!existsSync(modelPath)) {
             throw new Error(`Model '${modelName}' not found at ${modelPath}`);
@@ -422,16 +617,6 @@ export class SharedGGUFRunner {
             throw new Error(`Model '${modelName}' cannot be read at ${modelPath}`);
         }
     }
-    safeDispose(obj, name) {
-        if (!obj)
-            return;
-        try {
-            obj.dispose?.();
-        }
-        catch (err) {
-            getLogger().warn(`failed to dispose ${name}:`, err.message);
-        }
-    }
     logTransition(from, to, reason, extra) {
         getLogger().info(JSON.stringify({
             event: "runner_state_transition",
@@ -441,7 +626,7 @@ export class SharedGGUFRunner {
             reason,
             generation: this.generation,
             activeRequests: this.activeRequestCount,
-            contexts: this.contexts.size,
+            contexts: this.contextTrackers.size,
             timestamp: Date.now(),
             ...extra,
         }));