peerllm-host-cli 1.9.3 → 1.9.5

package/dist/core/capacity.d.ts

@@ -0,0 +1,62 @@
+/**
+ * Live host capacity snapshot sent to the orchestrator via `HeartbeatV2`.
+ *
+ * Field names mirror the Electron host's payload (camelCase) so both host kinds
+ * speak the identical contract to the server's `HostCapacity` model. The
+ * orchestrator routes around hosts whose fresh snapshot says they're out of
+ * VRAM / overloaded instead of piling work on until they OOM and crash.
+ */
+export interface HostCapacityPayload {
+    hostId: string;
+    utilizationType: string;
+    cpuUsage: number;
+    gpuUsage: number;
+    ramUsedMB: number;
+    ramFreeMB: number;
+    ramTotalMB: number;
+    vramFreeMB: number;
+    totalVRAM: number;
+    effectiveFreeMB: number;
+    activeSessions: number;
+    status: string;
+    canAcceptRequests: boolean;
+    lastUpdated: string;
+}
+/** Effective free memory for scheduling, ported verbatim from the Electron host. */
+export declare function calculateEffectiveMemory(args: {
+    vramFreeMB: number;
+    totalVRAM: number;
+    ramFreeMB: number;
+    utilizationType: string;
+}): number;
+/** Coarse host status, ported verbatim from the Electron host. */
+export declare function classifyStatus(args: {
+    utilizationType: string;
+    cpuUsage: number;
+    gpuUsage: number;
+    effectiveFreeMB: number;
+    vramFreeMB: number;
+    totalVRAM: number;
+}): string;
+/**
+ * Whether this host can take a new request.
+ *
+ * The 2 GB VRAM floor is applied ONLY to discrete-GPU hosts; CPU/Hybrid/unified
+ * hosts have vramFreeMB ≈ 0 and are judged purely on `status` (which already
+ * folds in CPU load and effective memory). This keeps CPU-only hosts eligible —
+ * "a place for every piece of hardware" — while preserving the 2 GB VRAM
+ * headroom for real GPU hosts. The Electron host's HeartbeatV2 computes
+ * canAcceptRequests with this same formula, so both host kinds stay in sync.
+ */
+export declare function computeCanAcceptRequests(args: {
+    utilizationType: string;
+    status: string;
+    vramFreeMB: number;
+}): boolean;
+/**
+ * Build the full HostCapacity snapshot for a HeartbeatV2 message. Best-effort:
+ * all sampling is wrapped so a probe failure degrades a field rather than
+ * throwing — the caller still gets a usable, conservative snapshot.
+ */
+export declare function buildHostCapacity(hostId: string, gpuIndex: number, activeSessions: number): Promise<HostCapacityPayload>;
+//# sourceMappingURL=capacity.d.ts.map

package/dist/core/capacity.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"capacity.d.ts","sourceRoot":"","sources":["../../src/core/capacity.ts"],"names":[],"mappings":"AAcA;;;;;;;GAOG;AACH,MAAM,WAAW,mBAAmB;IAClC,MAAM,EAAE,MAAM,CAAC;IACf,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,UAAU,EAAE,MAAM,CAAC;IACnB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;IACxB,cAAc,EAAE,MAAM,CAAC;IACvB,MAAM,EAAE,MAAM,CAAC;IACf,iBAAiB,EAAE,OAAO,CAAC;IAC3B,WAAW,EAAE,MAAM,CAAC;CACrB;AAqKD,oFAAoF;AACpF,wBAAgB,wBAAwB,CAAC,IAAI,EAAE;IAC7C,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;IAClB,SAAS,EAAE,MAAM,CAAC;IAClB,eAAe,EAAE,MAAM,CAAC;CACzB,GAAG,MAAM,CAST;AAED,kEAAkE;AAClE,wBAAgB,cAAc,CAAC,IAAI,EAAE;IACnC,eAAe,EAAE,MAAM,CAAC;IACxB,QAAQ,EAAE,MAAM,CAAC;IACjB,QAAQ,EAAE,MAAM,CAAC;IACjB,eAAe,EAAE,MAAM,CAAC;IACxB,UAAU,EAAE,MAAM,CAAC;IACnB,SAAS,EAAE,MAAM,CAAC;CACnB,GAAG,MAAM,CAOT;AAED;;;;;;;;;GASG;AACH,wBAAgB,wBAAwB,CAAC,IAAI,EAAE;IAC7C,eAAe,EAAE,MAAM,CAAC;IACxB,MAAM,EAAE,MAAM,CAAC;IACf,UAAU,EAAE,MAAM,CAAC;CACpB,GAAG,OAAO,CAKV;AAED;;;;GAIG;AACH,wBAAsB,iBAAiB,CACrC,MAAM,EAAE,MAAM,EACd,QAAQ,EAAE,MAAM,EAChB,cAAc,EAAE,MAAM,GACrB,OAAO,CAAC,mBAAmB,CAAC,CA8C9B"}

package/dist/core/capacity.js

@@ -0,0 +1,228 @@
+import { execFile } from "node:child_process";
+import { arch as osArch } from "node:os";
+import { promisify } from "node:util";
+import si from "systeminformation";
+import { sampleGpuUtilization } from "./gpu-stats.js";
+const execFileP = promisify(execFile);
+const NVIDIA_SMI_TIMEOUT_MS = 1500;
+const GRAPHICS_CACHE_TTL_MS = 5 * 60 * 1000;
+const isAppleSilicon = process.platform === "darwin" && osArch() === "arm64";
+// ---------------------------------------------------------------------------
+// Graphics cache — vendor/static VRAM rarely changes, and si.graphics() is the
+// expensive call. Live numbers (utilization, free VRAM) come from nvidia-smi /
+// the GPU sampler on each tick; the cache only supplies vendor + static totals.
+// ---------------------------------------------------------------------------
+let cachedGraphics;
+let graphicsInFlight;
+async function getCachedGraphics() {
+    const now = Date.now();
+    if (cachedGraphics && now - cachedGraphics.ts < GRAPHICS_CACHE_TTL_MS) {
+        return cachedGraphics.data;
+    }
+    if (!graphicsInFlight) {
+        graphicsInFlight = si.graphics().then((data) => {
+            cachedGraphics = { data, ts: Date.now() };
+            graphicsInFlight = undefined;
+            return data;
+        });
+    }
+    return graphicsInFlight;
+}
+/**
+ * Query the selected NVIDIA GPU for exact total/free VRAM and live utilization.
+ * `systeminformation`'s `memoryFree` is unreliable for NVIDIA on many systems,
+ * so — like the Electron host — we go straight to nvidia-smi for the numbers
+ * that drive the VRAM floor decision. Returns null when nvidia-smi is absent.
+ */
+async function queryNvidiaSmi(gpuIndex) {
+    try {
+        const cmd = process.platform === "win32"
+            ? `${process.env.SYSTEMROOT ?? "C:\\Windows"}\\System32\\nvidia-smi.exe`
+            : "nvidia-smi";
+        const { stdout } = await execFileP(cmd, [
+            "--query-gpu=memory.total,memory.free,utilization.gpu",
+            "--format=csv,noheader,nounits",
+        ], { timeout: NVIDIA_SMI_TIMEOUT_MS });
+        const lines = stdout.trim().split(/\r?\n/).filter((l) => l.trim().length > 0);
+        const line = lines[gpuIndex] ?? lines[0];
+        if (!line)
+            return null;
+        const [total, free, util] = line.split(/[\s,]+/).map((n) => parseInt(n, 10));
+        if (![total, free, util].every((n) => Number.isFinite(n)))
+            return null;
+        return { total: total, free: free, util: util };
+    }
+    catch {
+        return null;
+    }
+}
+/**
+ * Cross-platform GPU stats, ported from the Electron host's getGPUStats so both
+ * hosts classify utilization identically. Respects the user's selected GPU on
+ * multi-GPU systems (selectedGpuIndex), and treats unified-memory machines
+ * (Apple Silicon, AMD APUs) as RAM-backed rather than VRAM-backed.
+ */
+async function getGpuStats(mem, gpuIndex) {
+    const ginfo = await getCachedGraphics();
+    const controllers = ginfo.controllers ?? [];
+    const g = controllers[gpuIndex] ??
+        controllers[0] ??
+        {};
+    let gpuUsage = 0;
+    let totalVRAM = 0;
+    let vramFreeMB = 0;
+    let type = "CPU";
+    const ramTotalMB = mem.total / 1024 / 1024;
+    const ramFreeMB = mem.available / 1024 / 1024;
+    // 🍎 Apple Silicon — unified memory, no discrete VRAM.
+    if (isAppleSilicon) {
+        gpuUsage = g.utilizationGpu ?? 0;
+        if (!gpuUsage) {
+            const sampled = await sampleGpuUtilization();
+            if (sampled !== null)
+                gpuUsage = sampled * 100;
+        }
+        totalVRAM = ramTotalMB;
+        vramFreeMB = ramFreeMB;
+        // Under GPU pressure, discount free unified memory (floor at 50%).
+        vramFreeMB = Math.max(vramFreeMB * (1 - gpuUsage / 100), vramFreeMB * 0.5);
+        type = gpuUsage > 15 ? "Hybrid" : "CPU";
+        return { gpuUsage, totalVRAM, vramFreeMB, type };
+    }
+    const vendor = (g.vendor ?? "").toLowerCase();
+    const isAMD = vendor.includes("amd") || vendor.includes("ati") || vendor.includes("advanced micro");
+    const isIntel = vendor.includes("intel");
+    const isNvidia = vendor.includes("nvidia");
+    // 🟦 NVIDIA (Windows & Linux) — exact numbers from nvidia-smi.
+    if (isNvidia) {
+        const nv = await queryNvidiaSmi(gpuIndex);
+        if (nv) {
+            totalVRAM = nv.total;
+            vramFreeMB = nv.free;
+            gpuUsage = nv.util;
+            type = gpuUsage > 10 ? "GPU" : "CPU";
+            return { gpuUsage, totalVRAM, vramFreeMB, type };
+        }
+        // nvidia-smi unavailable → fall through to systeminformation values.
+    }
+    // 🟥 AMD (Windows & Linux).
+    if (isAMD) {
+        gpuUsage = g.utilizationGpu ?? 0;
+        totalVRAM = g.vram ?? g.memoryTotal ?? 0;
+        vramFreeMB = g.memoryFree ?? 0;
+        // Linux AMD APU (shared system RAM, e.g. Radeon 880M/890M) — treat as unified.
+        if (process.platform === "linux" && totalVRAM < 2048) {
+            totalVRAM = ramTotalMB;
+            vramFreeMB = ramFreeMB;
+            type = "Hybrid";
+            return { gpuUsage, totalVRAM, vramFreeMB, type };
+        }
+        // AMD driver quirk: free sometimes mirrors total — estimate from utilization.
+        if (vramFreeMB >= totalVRAM * 0.95) {
+            vramFreeMB = totalVRAM * (1 - gpuUsage / 100);
+        }
+        type = gpuUsage > 25 ? "GPU" : gpuUsage > 10 ? "Hybrid" : "CPU";
+        return { gpuUsage, totalVRAM, vramFreeMB, type };
+    }
+    // 🟫 Intel iGPU.
+    if (isIntel) {
+        gpuUsage = g.utilizationGpu ?? 0;
+        totalVRAM = g.vram ?? 0;
+        vramFreeMB = g.memoryFree ?? 0;
+        type = gpuUsage > 20 ? "Hybrid" : gpuUsage > 10 ? "CPU+GPU" : "CPU";
+        return { gpuUsage, totalVRAM, vramFreeMB, type };
+    }
+    // ⚪ Unknown / CPU-only.
+    gpuUsage = g.utilizationGpu ?? 0;
+    totalVRAM = g.vram ?? 0;
+    vramFreeMB = g.memoryFree ?? 0;
+    type = "CPU";
+    return { gpuUsage, totalVRAM, vramFreeMB, type };
+}
+/** Effective free memory for scheduling, ported verbatim from the Electron host. */
+export function calculateEffectiveMemory(args) {
+    const { vramFreeMB, totalVRAM, ramFreeMB, utilizationType } = args;
+    if (utilizationType === "Hybrid") {
+        return (vramFreeMB > 0 ? vramFreeMB : totalVRAM * 0.8) + ramFreeMB * 0.25;
+    }
+    if (utilizationType === "GPU" && totalVRAM > 0) {
+        return vramFreeMB || totalVRAM * 0.8;
+    }
+    return ramFreeMB;
+}
+/** Coarse host status, ported verbatim from the Electron host. */
+export function classifyStatus(args) {
+    const { utilizationType, cpuUsage, gpuUsage, effectiveFreeMB, vramFreeMB, totalVRAM } = args;
+    if (utilizationType === "GPU" && totalVRAM > 0 && vramFreeMB < 2000)
+        return "Overloaded";
+    if (cpuUsage > 85)
+        return "Busy";
+    if (gpuUsage > 90)
+        return "Busy";
+    if (effectiveFreeMB < 1500)
+        return "Overloaded";
+    return "Available";
+}
+/**
+ * Whether this host can take a new request.
+ *
+ * The 2 GB VRAM floor is applied ONLY to discrete-GPU hosts; CPU/Hybrid/unified
+ * hosts have vramFreeMB ≈ 0 and are judged purely on `status` (which already
+ * folds in CPU load and effective memory). This keeps CPU-only hosts eligible —
+ * "a place for every piece of hardware" — while preserving the 2 GB VRAM
+ * headroom for real GPU hosts. The Electron host's HeartbeatV2 computes
+ * canAcceptRequests with this same formula, so both host kinds stay in sync.
+ */
+export function computeCanAcceptRequests(args) {
+    const { utilizationType, status, vramFreeMB } = args;
+    if (status === "Overloaded" || status === "Busy")
+        return false;
+    if (utilizationType === "GPU" && vramFreeMB < 2000)
+        return false;
+    return true;
+}
+/**
+ * Build the full HostCapacity snapshot for a HeartbeatV2 message. Best-effort:
+ * all sampling is wrapped so a probe failure degrades a field rather than
+ * throwing — the caller still gets a usable, conservative snapshot.
+ */
+export async function buildHostCapacity(hostId, gpuIndex, activeSessions) {
+    const [cpu, mem] = await Promise.all([si.currentLoad(), si.mem()]);
+    const cpuUsage = cpu.currentLoad ?? 0;
+    const ramFreeMB = mem.available / 1024 / 1024;
+    const ramTotalMB = mem.total / 1024 / 1024;
+    const ramUsedMB = ramTotalMB - ramFreeMB;
+    const { gpuUsage, totalVRAM, vramFreeMB, type: utilizationType } = await getGpuStats(mem, gpuIndex);
+    const effectiveFreeMB = calculateEffectiveMemory({
+        vramFreeMB,
+        totalVRAM,
+        ramFreeMB,
+        utilizationType,
+    });
+    const status = classifyStatus({
+        utilizationType,
+        cpuUsage,
+        gpuUsage,
+        effectiveFreeMB,
+        vramFreeMB,
+        totalVRAM,
+    });
+    const canAcceptRequests = computeCanAcceptRequests({ utilizationType, status, vramFreeMB });
+    return {
+        hostId,
+        utilizationType,
+        cpuUsage,
+        gpuUsage,
+        ramUsedMB,
+        ramFreeMB,
+        ramTotalMB,
+        vramFreeMB,
+        totalVRAM,
+        effectiveFreeMB,
+        activeSessions,
+        status,
+        canAcceptRequests,
+        lastUpdated: new Date().toISOString(),
+    };
+}
+//# sourceMappingURL=capacity.js.map

package/dist/core/capacity.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"capacity.js","sourceRoot":"","sources":["../../src/core/capacity.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,MAAM,oBAAoB,CAAC;AAC9C,OAAO,EAAE,IAAI,IAAI,MAAM,EAAE,MAAM,SAAS,CAAC;AACzC,OAAO,EAAE,SAAS,EAAE,MAAM,WAAW,CAAC;AAEtC,OAAO,EAAE,MAAM,mBAAmB,CAAC;AAEnC,OAAO,EAAE,oBAAoB,EAAE,MAAM,gBAAgB,CAAC;AAEtD,MAAM,SAAS,GAAG,SAAS,CAAC,QAAQ,CAAC,CAAC;AACtC,MAAM,qBAAqB,GAAG,IAAI,CAAC;AACnC,MAAM,qBAAqB,GAAG,CAAC,GAAG,EAAE,GAAG,IAAI,CAAC;AAE5C,MAAM,cAAc,GAAG,OAAO,CAAC,QAAQ,KAAK,QAAQ,IAAI,MAAM,EAAE,KAAK,OAAO,CAAC;AA2B7E,8EAA8E;AAC9E,+EAA+E;AAC/E,+EAA+E;AAC/E,gFAAgF;AAChF,8EAA8E;AAE9E,IAAI,cAAmF,CAAC;AACxF,IAAI,gBAAwE,CAAC;AAE7E,KAAK,UAAU,iBAAiB;IAC9B,MAAM,GAAG,GAAG,IAAI,CAAC,GAAG,EAAE,CAAC;IACvB,IAAI,cAAc,IAAI,GAAG,GAAG,cAAc,CAAC,EAAE,GAAG,qBAAqB,EAAE,CAAC;QACtE,OAAO,cAAc,CAAC,IAAI,CAAC;IAC7B,CAAC;IACD,IAAI,CAAC,gBAAgB,EAAE,CAAC;QACtB,gBAAgB,GAAG,EAAE,CAAC,QAAQ,EAAE,CAAC,IAAI,CAAC,CAAC,IAAI,EAAE,EAAE;YAC7C,cAAc,GAAG,EAAE,IAAI,EAAE,EAAE,EAAE,IAAI,CAAC,GAAG,EAAE,EAAE,CAAC;YAC1C,gBAAgB,GAAG,SAAS,CAAC;YAC7B,OAAO,IAAI,CAAC;QACd,CAAC,CAAC,CAAC;IACL,CAAC;IACD,OAAO,gBAAgB,CAAC;AAC1B,CAAC;AAED;;;;;GAKG;AACH,KAAK,UAAU,cAAc,CAC3B,QAAgB;IAEhB,IAAI,CAAC;QACH,MAAM,GAAG,GACP,OAAO,CAAC,QAAQ,KAAK,OAAO;YAC1B,CAAC,CAAC,GAAG,OAAO,CAAC,GAAG,CAAC,UAAU,IAAI,aAAa,4BAA4B;YACxE,CAAC,CAAC,YAAY,CAAC;QACnB,MAAM,EAAE,MAAM,EAAE,GAAG,MAAM,SAAS,CAChC,GAAG,EACH;YACE,sDAAsD;YACtD,+BAA+B;SAChC,EACD,EAAE,OAAO,EAAE,qBAAqB,EAAE,CACnC,CAAC;QACF,MAAM,KAAK,GAAG,MAAM,CAAC,IAAI,EAAE,CAAC,KAAK,CAAC,OAAO,CAAC,CAAC,MAAM,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC,IAAI,EAAE,CAAC,MAAM,GAAG,CAAC,CAAC,CAAC;QAC9E,MAAM,IAAI,GAAG,KAAK,CAAC,QAAQ,CAAC,IAAI,KAAK,CAAC,CAAC,CAAC,CAAC;QACzC,IAAI,CAAC,IAAI;YAAE,OAAO,IAAI,CAAC;QACvB,MAAM,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,GAAG,IAAI,CAAC,KAAK,CAAC,QAAQ,CAAC,CAAC,GAAG,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,QAAQ,CAAC,CAAC,EAAE,EAAE,CAAC,CAAC,CAAC;QAC7E,IAAI,CAAC,CAAC,KAAK,EAAE,IAAI,EAAE,IAAI,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,EAAE,EAAE,CAAC,MAAM,CAAC,QAAQ,CAAC,CAAC,CAAC,CAAC;YAAE,OAAO,IAAI,CAAC;QACvE,OAAO,EAAE,KAAK,EAAE,KAAM,EAAE,IAAI,EAAE,IAAK,EAAE,IAAI,EAAE,IAAK,EAAE,CAAC;IACrD,CAAC;IAAC,MAAM,CAAC;QACP,OAAO,IAAI,CAAC;IACd,CAAC;AACH,CAAC;AASD;;;;;GAKG;AACH,KAAK,UAAU,WAAW,CACxB,GAAiC,EACjC,QAAgB;IAEhB,MAAM,KAAK,GAAG,MAAM,iBAAiB,EAAE,CAAC;IACxC,MAAM,WAAW,GAAG,KAAK,CAAC,WAAW,IAAI,EAAE,CAAC;IAC5C,MAAM,CAAC,GACL,WAAW,CAAC,QAAQ,CAAC;QACrB,WAAW,CAAC,CAAC,CAAC;QACb,EAAkD,CAAC;IAEtD,IAAI,QAAQ,GAAG,CAAC,CAAC;IACjB,IAAI,SAAS,GAAG,CAAC,CAAC;IAClB,IAAI,UAAU,GAAG,CAAC,CAAC;IACnB,IAAI,IAAI,GAAG,KAAK,CAAC;IAEjB,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3C,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC;IAE9C,uDAAuD;IACvD,IAAI,cAAc,EAAE,CAAC;QACnB,QAAQ,GAAG,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;QACjC,IAAI,CAAC,QAAQ,EAAE,CAAC;YACd,MAAM,OAAO,GAAG,MAAM,oBAAoB,EAAE,CAAC;YAC7C,IAAI,OAAO,KAAK,IAAI;gBAAE,QAAQ,GAAG,OAAO,GAAG,GAAG,CAAC;QACjD,CAAC;QACD,SAAS,GAAG,UAAU,CAAC;QACvB,UAAU,GAAG,SAAS,CAAC;QACvB,mEAAmE;QACnE,UAAU,GAAG,IAAI,CAAC,GAAG,CAAC,UAAU,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,EAAE,UAAU,GAAG,GAAG,CAAC,CAAC;QAC3E,IAAI,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;QACxC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,MAAM,MAAM,GAAG,CAAC,CAAC,CAAC,MAAM,IAAI,EAAE,CAAC,CAAC,WAAW,EAAE,CAAC;IAC9C,MAAM,KAAK,GACT,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,KAAK,CAAC,IAAI,MAAM,CAAC,QAAQ,CAAC,gBAAgB,CAAC,CAAC;IACxF,MAAM,OAAO,GAAG,MAAM,CAAC,QAAQ,CAAC,OAAO,CAAC,CAAC;IACzC,MAAM,QAAQ,GAAG,MAAM,CAAC,QAAQ,CAAC,QAAQ,CAAC,CAAC;IAE3C,+DAA+D;IAC/D,IAAI,QAAQ,EAAE,CAAC;QACb,MAAM,EAAE,GAAG,MAAM,cAAc,CAAC,QAAQ,CAAC,CAAC;QAC1C,IAAI,EAAE,EAAE,CAAC;YACP,SAAS,GAAG,EAAE,CAAC,KAAK,CAAC;YACrB,UAAU,GAAG,EAAE,CAAC,IAAI,CAAC;YACrB,QAAQ,GAAG,EAAE,CAAC,IAAI,CAAC;YACnB,IAAI,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,KAAK,CAAC;YACrC,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACnD,CAAC;QACD,qEAAqE;IACvE,CAAC;IAED,4BAA4B;IAC5B,IAAI,KAAK,EAAE,CAAC;QACV,QAAQ,GAAG,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;QACjC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC,WAAW,IAAI,CAAC,CAAC;QACzC,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;QAE/B,+EAA+E;QAC/E,IAAI,OAAO,CAAC,QAAQ,KAAK,OAAO,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACrD,SAAS,GAAG,UAAU,CAAC;YACvB,UAAU,GAAG,SAAS,CAAC;YACvB,IAAI,GAAG,QAAQ,CAAC;YAChB,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;QACnD,CAAC;QAED,8EAA8E;QAC9E,IAAI,UAAU,IAAI,SAAS,GAAG,IAAI,EAAE,CAAC;YACnC,UAAU,GAAG,SAAS,GAAG,CAAC,CAAC,GAAG,QAAQ,GAAG,GAAG,CAAC,CAAC;QAChD,CAAC;QAED,IAAI,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,KAAK,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,KAAK,CAAC;QAChE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,iBAAiB;IACjB,IAAI,OAAO,EAAE,CAAC;QACZ,QAAQ,GAAG,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;QACjC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;QACxB,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;QAC/B,IAAI,GAAG,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,QAAQ,CAAC,CAAC,CAAC,QAAQ,GAAG,EAAE,CAAC,CAAC,CAAC,SAAS,CAAC,CAAC,CAAC,KAAK,CAAC;QACpE,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;IACnD,CAAC;IAED,wBAAwB;IACxB,QAAQ,GAAG,CAAC,CAAC,cAAc,IAAI,CAAC,CAAC;IACjC,SAAS,GAAG,CAAC,CAAC,IAAI,IAAI,CAAC,CAAC;IACxB,UAAU,GAAG,CAAC,CAAC,UAAU,IAAI,CAAC,CAAC;IAC/B,IAAI,GAAG,KAAK,CAAC;IACb,OAAO,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,CAAC;AACnD,CAAC;AAED,oFAAoF;AACpF,MAAM,UAAU,wBAAwB,CAAC,IAKxC;IACC,MAAM,EAAE,UAAU,EAAE,SAAS,EAAE,SAAS,EAAE,eAAe,EAAE,GAAG,IAAI,CAAC;IACnE,IAAI,eAAe,KAAK,QAAQ,EAAE,CAAC;QACjC,OAAO,CAAC,UAAU,GAAG,CAAC,CAAC,CAAC,CAAC,UAAU,CAAC,CAAC,CAAC,SAAS,GAAG,GAAG,CAAC,GAAG,SAAS,GAAG,IAAI,CAAC;IAC5E,CAAC;IACD,IAAI,eAAe,KAAK,KAAK,IAAI,SAAS,GAAG,CAAC,EAAE,CAAC;QAC/C,OAAO,UAAU,IAAI,SAAS,GAAG,GAAG,CAAC;IACvC,CAAC;IACD,OAAO,SAAS,CAAC;AACnB,CAAC;AAED,kEAAkE;AAClE,MAAM,UAAU,cAAc,CAAC,IAO9B;IACC,MAAM,EAAE,eAAe,EAAE,QAAQ,EAAE,QAAQ,EAAE,eAAe,EAAE,UAAU,EAAE,SAAS,EAAE,GAAG,IAAI,CAAC;IAC7F,IAAI,eAAe,KAAK,KAAK,IAAI,SAAS,GAAG,CAAC,IAAI,UAAU,GAAG,IAAI;QAAE,OAAO,YAAY,CAAC;IACzF,IAAI,QAAQ,GAAG,EAAE;QAAE,OAAO,MAAM,CAAC;IACjC,IAAI,QAAQ,GAAG,EAAE;QAAE,OAAO,MAAM,CAAC;IACjC,IAAI,eAAe,GAAG,IAAI;QAAE,OAAO,YAAY,CAAC;IAChD,OAAO,WAAW,CAAC;AACrB,CAAC;AAED;;;;;;;;;GASG;AACH,MAAM,UAAU,wBAAwB,CAAC,IAIxC;IACC,MAAM,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,GAAG,IAAI,CAAC;IACrD,IAAI,MAAM,KAAK,YAAY,IAAI,MAAM,KAAK,MAAM;QAAE,OAAO,KAAK,CAAC;IAC/D,IAAI,eAAe,KAAK,KAAK,IAAI,UAAU,GAAG,IAAI;QAAE,OAAO,KAAK,CAAC;IACjE,OAAO,IAAI,CAAC;AACd,CAAC;AAED;;;;GAIG;AACH,MAAM,CAAC,KAAK,UAAU,iBAAiB,CACrC,MAAc,EACd,QAAgB,EAChB,cAAsB;IAEtB,MAAM,CAAC,GAAG,EAAE,GAAG,CAAC,GAAG,MAAM,OAAO,CAAC,GAAG,CAAC,CAAC,EAAE,CAAC,WAAW,EAAE,EAAE,EAAE,CAAC,GAAG,EAAE,CAAC,CAAC,CAAC;IACnE,MAAM,QAAQ,GAAG,GAAG,CAAC,WAAW,IAAI,CAAC,CAAC;IACtC,MAAM,SAAS,GAAG,GAAG,CAAC,SAAS,GAAG,IAAI,GAAG,IAAI,CAAC;IAC9C,MAAM,UAAU,GAAG,GAAG,CAAC,KAAK,GAAG,IAAI,GAAG,IAAI,CAAC;IAC3C,MAAM,SAAS,GAAG,UAAU,GAAG,SAAS,CAAC;IAEzC,MAAM,EAAE,QAAQ,EAAE,SAAS,EAAE,UAAU,EAAE,IAAI,EAAE,eAAe,EAAE,GAAG,MAAM,WAAW,CAClF,GAAG,EACH,QAAQ,CACT,CAAC;IAEF,MAAM,eAAe,GAAG,wBAAwB,CAAC;QAC/C,UAAU;QACV,SAAS;QACT,SAAS;QACT,eAAe;KAChB,CAAC,CAAC;IAEH,MAAM,MAAM,GAAG,cAAc,CAAC;QAC5B,eAAe;QACf,QAAQ;QACR,QAAQ;QACR,eAAe;QACf,UAAU;QACV,SAAS;KACV,CAAC,CAAC;IAEH,MAAM,iBAAiB,GAAG,wBAAwB,CAAC,EAAE,eAAe,EAAE,MAAM,EAAE,UAAU,EAAE,CAAC,CAAC;IAE5F,OAAO;QACL,MAAM;QACN,eAAe;QACf,QAAQ;QACR,QAAQ;QACR,SAAS;QACT,SAAS;QACT,UAAU;QACV,UAAU;QACV,SAAS;QACT,eAAe;QACf,cAAc;QACd,MAAM;QACN,iBAAiB;QACjB,WAAW,EAAE,IAAI,IAAI,EAAE,CAAC,WAAW,EAAE;KACtC,CAAC;AACJ,CAAC"}

package/dist/core/llmWorker.d.ts

@@ -0,0 +1,2 @@
+export {};
+//# sourceMappingURL=llmWorker.d.ts.map

package/dist/core/llmWorker.d.ts.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmWorker.d.ts","sourceRoot":"","sources":["../../src/core/llmWorker.ts"],"names":[],"mappings":""}

package/dist/core/llmWorker.js

@@ -0,0 +1,239 @@
+import { getLlama, LlamaChatSession } from "node-llama-cpp";
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let llama = null;
+// eslint-disable-next-line @typescript-eslint/no-explicit-any
+let model = null;
+const contexts = new Map();
+function send(msg) {
+    process.send?.(msg);
+}
+function fwdLog(level, message) {
+    send({ type: "log", level, message });
+}
+console.log = (...args) => fwdLog("info", args.map(String).join(" "));
+console.warn = (...args) => fwdLog("warn", args.map(String).join(" "));
+console.error = (...args) => fwdLog("error", args.map(String).join(" "));
+process.on("uncaughtException", (err) => {
+    fwdLog("error", `llmWorker uncaughtException: ${err.message}`);
+    process.exit(1);
+});
+process.on("unhandledRejection", (reason) => {
+    fwdLog("error", `llmWorker unhandledRejection: ${String(reason)}`);
+});
+function cancelAll() {
+    for (const entry of contexts.values()) {
+        if (entry.abortController) {
+            try {
+                entry.abortController.abort();
+            }
+            catch {
+                // ignored
+            }
+        }
+    }
+}
+process.on("message", (msg) => {
+    const m = msg;
+    const type = m?.["type"];
+    void (async () => {
+        try {
+            switch (type) {
+                case "load-model": {
+                    const modelPath = m["modelPath"];
+                    const gpuLayers = m["gpuLayers"];
+                    try {
+                        llama = await getLlama();
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                        model = await llama.loadModel({ modelPath, gpuLayers });
+                    }
+                    catch (err) {
+                        const errAny = err;
+                        if (errAny?.constructor?.name === "InsufficientMemoryError") {
+                            try {
+                                // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                                model = await llama.loadModel({
+                                    modelPath,
+                                    gpuLayers,
+                                    ignoreMemorySafetyChecks: true,
+                                });
+                            }
+                            catch (err2) {
+                                send({ type: "error", message: err2.message });
+                                break;
+                            }
+                        }
+                        else {
+                            send({ type: "error", message: err.message });
+                            break;
+                        }
+                    }
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+                    const trainContextSize = model?.trainContextSize ?? 4096;
+                    send({ type: "loaded", trainContextSize });
+                    break;
+                }
+                case "create-context": {
+                    const conversationId = m["conversationId"];
+                    try {
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-member-access
+                        const trainSize = model?.trainContextSize ?? 4096;
+                        const contextSize = Math.min(trainSize, 4096);
+                        // eslint-disable-next-line @typescript-eslint/no-explicit-any
+                        let context;
+                        try {
+                            // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                            context = await model.createContext({ contextSize });
+                        }
+                        catch (err) {
+                            const errAny = err;
+                            if (errAny?.constructor?.name === "InsufficientMemoryError") {
+                                // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                                context = await model.createContext({ contextSize, ignoreMemorySafetyChecks: true });
+                            }
+                            else {
+                                throw err;
+                            }
+                        }
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                        const session = new LlamaChatSession({ contextSequence: context.getSequence() });
+                        contexts.set(conversationId, { context, session, abortController: null });
+                        send({ type: "context-created", conversationId });
+                    }
+                    catch (err) {
+                        send({ type: "error", conversationId, message: err.message });
+                    }
+                    break;
+                }
+                case "clear-context": {
+                    const conversationId = m["conversationId"];
+                    const entry = contexts.get(conversationId);
+                    if (entry) {
+                        if (entry.abortController) {
+                            try {
+                                entry.abortController.abort();
+                            }
+                            catch {
+                                // ignored
+                            }
+                        }
+                        contexts.delete(conversationId);
+                        try {
+                            // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                            entry.context?.dispose?.();
+                        }
+                        catch {
+                            // ignored
+                        }
+                    }
+                    break;
+                }
+                case "prompt": {
+                    const requestId = m["requestId"];
+                    const conversationId = m["conversationId"];
+                    const input = m["input"];
+                    const options = m["options"];
+                    const entry = contexts.get(conversationId);
+                    if (!entry) {
+                        send({ type: "error", requestId, message: `No context for "${conversationId}"` });
+                        break;
+                    }
+                    entry.abortController = new AbortController();
+                    // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                    const promptTokens = model.tokenize(input).length;
+                    let fullText = "";
+                    try {
+                        const promptOpts = {
+                            onTextChunk: (chunk) => {
+                                fullText += chunk;
+                                send({ type: "token", requestId, chunk });
+                            },
+                            signal: entry.abortController.signal,
+                        };
+                        if (options?.maxTokens !== undefined)
+                            promptOpts["maxTokens"] = options.maxTokens;
+                        if (options?.temperature !== undefined)
+                            promptOpts["temperature"] = options.temperature;
+                        if (options?.stop?.length)
+                            promptOpts["customStopTriggers"] = options.stop;
+                        await entry.session.prompt(input, promptOpts);
+                        fullText = fullText.trim();
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                        const completionTokens = model.tokenize(fullText).length;
+                        send({ type: "done", requestId, text: fullText, promptTokens, completionTokens });
+                    }
+                    catch (err) {
+                        const e = err;
+                        if (e?.name === "AbortError") {
+                            // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                            const completionTokens = model ? model.tokenize(fullText).length : 0;
+                            send({ type: "cancelled", requestId, text: fullText, promptTokens, completionTokens });
+                        }
+                        else {
+                            send({ type: "error", requestId, message: err.message });
+                        }
+                    }
+                    finally {
+                        entry.abortController = null;
+                    }
+                    break;
+                }
+                case "cancel": {
+                    const entry = contexts.get(m["conversationId"]);
+                    if (entry?.abortController) {
+                        try {
+                            entry.abortController.abort();
+                        }
+                        catch {
+                            // ignored
+                        }
+                    }
+                    break;
+                }
+                case "cancel-all": {
+                    cancelAll();
+                    break;
+                }
+                case "dispose": {
+                    cancelAll();
+                    await new Promise((r) => setTimeout(r, 300));
+                    for (const entry of contexts.values()) {
+                        try {
+                            // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                            entry.context?.dispose?.();
+                        }
+                        catch {
+                            // ignored
+                        }
+                    }
+                    contexts.clear();
+                    try {
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                        model?.dispose?.();
+                    }
+                    catch {
+                        // ignored
+                    }
+                    try {
+                        // eslint-disable-next-line @typescript-eslint/no-unsafe-call, @typescript-eslint/no-unsafe-member-access
+                        llama?.dispose?.();
+                    }
+                    catch {
+                        // ignored
+                    }
+                    model = null;
+                    llama = null;
+                    send({ type: "disposed" });
+                    process.exit(0);
+                    break;
+                }
+                default:
+                    fwdLog("warn", `llmWorker: unknown message type "${type ?? "(none)"}"`);
+            }
+        }
+        catch (err) {
+            fwdLog("error", `llmWorker error in "${type}": ${err.message}`);
+        }
+    })();
+});
+send({ type: "ready" });
+//# sourceMappingURL=llmWorker.js.map

package/dist/core/llmWorker.js.map

@@ -0,0 +1 @@
+{"version":3,"file":"llmWorker.js","sourceRoot":"","sources":["../../src/core/llmWorker.ts"],"names":[],"mappings":"AAAA,OAAO,EAAE,QAAQ,EAAE,gBAAgB,EAAE,MAAM,gBAAgB,CAAC;AAW5D,8DAA8D;AAC9D,IAAI,KAAK,GAAQ,IAAI,CAAC;AACtB,8DAA8D;AAC9D,IAAI,KAAK,GAAQ,IAAI,CAAC;AACtB,MAAM,QAAQ,GAAG,IAAI,GAAG,EAAwB,CAAC;AAEjD,SAAS,IAAI,CAAC,GAAY;IACxB,OAAO,CAAC,IAAI,EAAE,CAAC,GAAG,CAAC,CAAC;AACtB,CAAC;AAED,SAAS,MAAM,CAAC,KAAe,EAAE,OAAe;IAC9C,IAAI,CAAC,EAAE,IAAI,EAAE,KAAK,EAAE,KAAK,EAAE,OAAO,EAAE,CAAC,CAAC;AACxC,CAAC;AAED,OAAO,CAAC,GAAG,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AACjF,OAAO,CAAC,IAAI,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,MAAM,CAAC,MAAM,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAClF,OAAO,CAAC,KAAK,GAAG,CAAC,GAAG,IAAe,EAAE,EAAE,CAAC,MAAM,CAAC,OAAO,EAAE,IAAI,CAAC,GAAG,CAAC,MAAM,CAAC,CAAC,IAAI,CAAC,GAAG,CAAC,CAAC,CAAC;AAEpF,OAAO,CAAC,EAAE,CAAC,mBAAmB,EAAE,CAAC,GAAU,EAAE,EAAE;IAC7C,MAAM,CAAC,OAAO,EAAE,gCAAgC,GAAG,CAAC,OAAO,EAAE,CAAC,CAAC;IAC/D,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;AAClB,CAAC,CAAC,CAAC;AAEH,OAAO,CAAC,EAAE,CAAC,oBAAoB,EAAE,CAAC,MAAe,EAAE,EAAE;IACnD,MAAM,CAAC,OAAO,EAAE,iCAAiC,MAAM,CAAC,MAAM,CAAC,EAAE,CAAC,CAAC;AACrE,CAAC,CAAC,CAAC;AAEH,SAAS,SAAS;IAChB,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;QACtC,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;YAC1B,IAAI,CAAC;gBACH,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;YAChC,CAAC;YAAC,MAAM,CAAC;gBACP,UAAU;YACZ,CAAC;QACH,CAAC;IACH,CAAC;AACH,CAAC;AAED,OAAO,CAAC,EAAE,CAAC,SAAS,EAAE,CAAC,GAAY,EAAE,EAAE;IACrC,MAAM,CAAC,GAAG,GAA8B,CAAC;IACzC,MAAM,IAAI,GAAG,CAAC,EAAE,CAAC,MAAM,CAAuB,CAAC;IAE/C,KAAK,CAAC,KAAK,IAAI,EAAE;QACf,IAAI,CAAC;YACH,QAAQ,IAAI,EAAE,CAAC;gBACb,KAAK,YAAY,CAAC,CAAC,CAAC;oBAClB,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAW,CAAC;oBAC3C,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAW,CAAC;oBAC3C,IAAI,CAAC;wBACH,KAAK,GAAG,MAAM,QAAQ,EAAE,CAAC;wBACzB,yGAAyG;wBACzG,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC,EAAE,SAAS,EAAE,SAAS,EAAE,CAAC,CAAC;oBAC1D,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,MAAM,GAAG,GAA0C,CAAC;wBAC1D,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,KAAK,yBAAyB,EAAE,CAAC;4BAC5D,IAAI,CAAC;gCACH,yGAAyG;gCACzG,KAAK,GAAG,MAAM,KAAK,CAAC,SAAS,CAAC;oCAC5B,SAAS;oCACT,SAAS;oCACT,wBAAwB,EAAE,IAAI;iCAC/B,CAAC,CAAC;4BACL,CAAC;4BAAC,OAAO,IAAI,EAAE,CAAC;gCACd,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAG,IAAc,CAAC,OAAO,EAAE,CAAC,CAAC;gCAC1D,MAAM;4BACR,CAAC;wBACH,CAAC;6BAAM,CAAC;4BACN,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;4BACzD,MAAM;wBACR,CAAC;oBACH,CAAC;oBACD,sEAAsE;oBACtE,MAAM,gBAAgB,GAAI,KAAK,EAAE,gBAAuC,IAAI,IAAI,CAAC;oBACjF,IAAI,CAAC,EAAE,IAAI,EAAE,QAAQ,EAAE,gBAAgB,EAAE,CAAC,CAAC;oBAC3C,MAAM;gBACR,CAAC;gBAED,KAAK,gBAAgB,CAAC,CAAC,CAAC;oBACtB,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAgB,CAAW,CAAC;oBACrD,IAAI,CAAC;wBACH,sEAAsE;wBACtE,MAAM,SAAS,GAAI,KAAK,EAAE,gBAAuC,IAAI,IAAI,CAAC;wBAC1E,MAAM,WAAW,GAAG,IAAI,CAAC,GAAG,CAAC,SAAS,EAAE,IAAI,CAAC,CAAC;wBAC9C,8DAA8D;wBAC9D,IAAI,OAAY,CAAC;wBACjB,IAAI,CAAC;4BACH,yGAAyG;4BACzG,OAAO,GAAG,MAAM,KAAK,CAAC,aAAa,CAAC,EAAE,WAAW,EAAE,CAAC,CAAC;wBACvD,CAAC;wBAAC,OAAO,GAAG,EAAE,CAAC;4BACb,MAAM,MAAM,GAAG,GAA0C,CAAC;4BAC1D,IAAI,MAAM,EAAE,WAAW,EAAE,IAAI,KAAK,yBAAyB,EAAE,CAAC;gCAC5D,yGAAyG;gCACzG,OAAO,GAAG,MAAM,KAAK,CAAC,aAAa,CAAC,EAAE,WAAW,EAAE,wBAAwB,EAAE,IAAI,EAAE,CAAC,CAAC;4BACvF,CAAC;iCAAM,CAAC;gCACN,MAAM,GAAG,CAAC;4BACZ,CAAC;wBACH,CAAC;wBACD,yGAAyG;wBACzG,MAAM,OAAO,GAAG,IAAI,gBAAgB,CAAC,EAAE,eAAe,EAAE,OAAO,CAAC,WAAW,EAAE,EAAE,CAAC,CAAC;wBACjF,QAAQ,CAAC,GAAG,CAAC,cAAc,EAAE,EAAE,OAAO,EAAE,OAAO,EAAE,eAAe,EAAE,IAAI,EAAE,CAAC,CAAC;wBAC1E,IAAI,CAAC,EAAE,IAAI,EAAE,iBAAiB,EAAE,cAAc,EAAE,CAAC,CAAC;oBACpD,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,cAAc,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;oBAC3E,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,KAAK,eAAe,CAAC,CAAC,CAAC;oBACrB,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAgB,CAAW,CAAC;oBACrD,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;oBAC3C,IAAI,KAAK,EAAE,CAAC;wBACV,IAAI,KAAK,CAAC,eAAe,EAAE,CAAC;4BAC1B,IAAI,CAAC;gCACH,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;4BAChC,CAAC;4BAAC,MAAM,CAAC;gCACP,UAAU;4BACZ,CAAC;wBACH,CAAC;wBACD,QAAQ,CAAC,MAAM,CAAC,cAAc,CAAC,CAAC;wBAChC,IAAI,CAAC;4BACH,yGAAyG;4BACzG,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;wBAC7B,CAAC;wBAAC,MAAM,CAAC;4BACP,UAAU;wBACZ,CAAC;oBACH,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACd,MAAM,SAAS,GAAG,CAAC,CAAC,WAAW,CAAW,CAAC;oBAC3C,MAAM,cAAc,GAAG,CAAC,CAAC,gBAAgB,CAAW,CAAC;oBACrD,MAAM,KAAK,GAAG,CAAC,CAAC,OAAO,CAAW,CAAC;oBACnC,MAAM,OAAO,GAAG,CAAC,CAAC,SAAS,CAEd,CAAC;oBAEd,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,cAAc,CAAC,CAAC;oBAC3C,IAAI,CAAC,KAAK,EAAE,CAAC;wBACX,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAE,mBAAmB,cAAc,GAAG,EAAE,CAAC,CAAC;wBAClF,MAAM;oBACR,CAAC;oBAED,KAAK,CAAC,eAAe,GAAG,IAAI,eAAe,EAAE,CAAC;oBAC9C,yGAAyG;oBACzG,MAAM,YAAY,GAAI,KAAK,CAAC,QAAQ,CAAC,KAAK,CAAe,CAAC,MAAM,CAAC;oBACjE,IAAI,QAAQ,GAAG,EAAE,CAAC;oBAElB,IAAI,CAAC;wBACH,MAAM,UAAU,GAA4B;4BAC1C,WAAW,EAAE,CAAC,KAAa,EAAE,EAAE;gCAC7B,QAAQ,IAAI,KAAK,CAAC;gCAClB,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,KAAK,EAAE,CAAC,CAAC;4BAC5C,CAAC;4BACD,MAAM,EAAE,KAAK,CAAC,eAAe,CAAC,MAAM;yBACrC,CAAC;wBACF,IAAI,OAAO,EAAE,SAAS,KAAK,SAAS;4BAAE,UAAU,CAAC,WAAW,CAAC,GAAG,OAAO,CAAC,SAAS,CAAC;wBAClF,IAAI,OAAO,EAAE,WAAW,KAAK,SAAS;4BAAE,UAAU,CAAC,aAAa,CAAC,GAAG,OAAO,CAAC,WAAW,CAAC;wBACxF,IAAI,OAAO,EAAE,IAAI,EAAE,MAAM;4BAAE,UAAU,CAAC,oBAAoB,CAAC,GAAG,OAAO,CAAC,IAAI,CAAC;wBAE3E,MAAM,KAAK,CAAC,OAAO,CAAC,MAAM,CAAC,KAAK,EAAE,UAAU,CAAC,CAAC;wBAE9C,QAAQ,GAAG,QAAQ,CAAC,IAAI,EAAE,CAAC;wBAC3B,yGAAyG;wBACzG,MAAM,gBAAgB,GAAI,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAe,CAAC,MAAM,CAAC;wBACxE,IAAI,CAAC,EAAE,IAAI,EAAE,MAAM,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;oBACpF,CAAC;oBAAC,OAAO,GAAG,EAAE,CAAC;wBACb,MAAM,CAAC,GAAG,GAAwB,CAAC;wBACnC,IAAI,CAAC,EAAE,IAAI,KAAK,YAAY,EAAE,CAAC;4BAC7B,yGAAyG;4BACzG,MAAM,gBAAgB,GAAG,KAAK,CAAC,CAAC,CAAE,KAAK,CAAC,QAAQ,CAAC,QAAQ,CAAe,CAAC,MAAM,CAAC,CAAC,CAAC,CAAC,CAAC;4BACpF,IAAI,CAAC,EAAE,IAAI,EAAE,WAAW,EAAE,SAAS,EAAE,IAAI,EAAE,QAAQ,EAAE,YAAY,EAAE,gBAAgB,EAAE,CAAC,CAAC;wBACzF,CAAC;6BAAM,CAAC;4BACN,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,SAAS,EAAE,OAAO,EAAG,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;wBACtE,CAAC;oBACH,CAAC;4BAAS,CAAC;wBACT,KAAK,CAAC,eAAe,GAAG,IAAI,CAAC;oBAC/B,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,KAAK,QAAQ,CAAC,CAAC,CAAC;oBACd,MAAM,KAAK,GAAG,QAAQ,CAAC,GAAG,CAAC,CAAC,CAAC,gBAAgB,CAAW,CAAC,CAAC;oBAC1D,IAAI,KAAK,EAAE,eAAe,EAAE,CAAC;wBAC3B,IAAI,CAAC;4BACH,KAAK,CAAC,eAAe,CAAC,KAAK,EAAE,CAAC;wBAChC,CAAC;wBAAC,MAAM,CAAC;4BACP,UAAU;wBACZ,CAAC;oBACH,CAAC;oBACD,MAAM;gBACR,CAAC;gBAED,KAAK,YAAY,CAAC,CAAC,CAAC;oBAClB,SAAS,EAAE,CAAC;oBACZ,MAAM;gBACR,CAAC;gBAED,KAAK,SAAS,CAAC,CAAC,CAAC;oBACf,SAAS,EAAE,CAAC;oBACZ,MAAM,IAAI,OAAO,CAAO,CAAC,CAAC,EAAE,EAAE,CAAC,UAAU,CAAC,CAAC,EAAE,GAAG,CAAC,CAAC,CAAC;oBACnD,KAAK,MAAM,KAAK,IAAI,QAAQ,CAAC,MAAM,EAAE,EAAE,CAAC;wBACtC,IAAI,CAAC;4BACH,yGAAyG;4BACzG,KAAK,CAAC,OAAO,EAAE,OAAO,EAAE,EAAE,CAAC;wBAC7B,CAAC;wBAAC,MAAM,CAAC;4BACP,UAAU;wBACZ,CAAC;oBACH,CAAC;oBACD,QAAQ,CAAC,KAAK,EAAE,CAAC;oBACjB,IAAI,CAAC;wBACH,yGAAyG;wBACzG,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC;wBACP,UAAU;oBACZ,CAAC;oBACD,IAAI,CAAC;wBACH,yGAAyG;wBACzG,KAAK,EAAE,OAAO,EAAE,EAAE,CAAC;oBACrB,CAAC;oBAAC,MAAM,CAAC;wBACP,UAAU;oBACZ,CAAC;oBACD,KAAK,GAAG,IAAI,CAAC;oBACb,KAAK,GAAG,IAAI,CAAC;oBACb,IAAI,CAAC,EAAE,IAAI,EAAE,UAAU,EAAE,CAAC,CAAC;oBAC3B,OAAO,CAAC,IAAI,CAAC,CAAC,CAAC,CAAC;oBAChB,MAAM;gBACR,CAAC;gBAED;oBACE,MAAM,CAAC,MAAM,EAAE,oCAAoC,IAAI,IAAI,QAAQ,GAAG,CAAC,CAAC;YAC5E,CAAC;QACH,CAAC;QAAC,OAAO,GAAG,EAAE,CAAC;YACb,MAAM,CAAC,OAAO,EAAE,uBAAuB,IAAI,MAAO,GAAa,CAAC,OAAO,EAAE,CAAC,CAAC;QAC7E,CAAC;IACH,CAAC,CAAC,EAAE,CAAC;AACP,CAAC,CAAC,CAAC;AAEH,IAAI,CAAC,EAAE,IAAI,EAAE,OAAO,EAAE,CAAC,CAAC"}

package/dist/core/orchestrator.d.ts.map

@@ -1 +1 @@
-{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AASA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAC;AA2BxE,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;AAErG,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,mBAAmB;IAkBlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAjBjC,OAAO,CAAC,UAAU,CAA8B;IAChD,OAAO,CAAC,iBAAiB,CAA+B;IACxD,OAAO,CAAC,eAAe,CAA+B;IACtD,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,MAAM,CAAsC;IAEpD,OAAO,CAAC,iBAAiB,CAAS;IAElC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgE;IACzF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA0B;IACpD,OAAO,CAAC,cAAc,CAAgB;gBAET,IAAI,EAAE,mBAAmB;IAQtD,SAAS,IAAI,kBAAkB;IAI/B,cAAc,IAAI;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;IAIvG,aAAa,IAAI;QAAE,EAAE,EAAE,cAAc,CAAC;QAAC,GAAG,EAAE,cAAc,CAAA;KAAE;IAI5D,iBAAiB,IAAI,MAAM,EAAE;IAIvB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB/C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAsHxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAmBnB,gBAAgB;IA4B9B,OAAO,CAAC,iBAAiB;IAanB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;YA2B5B,yBAAyB;YAkCzB,YAAY;IA0E1B,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,UAAU;YAeJ,IAAI;IASlB,OAAO,CAAC,SAAS;CAKlB"}
+{"version":3,"file":"orchestrator.d.ts","sourceRoot":"","sources":["../../src/core/orchestrator.ts"],"names":[],"mappings":"AAUA,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,aAAa,CAAC;AACjD,OAAO,EAAE,cAAc,EAAE,MAAM,aAAa,CAAC;AAG7C,OAAO,KAAK,EAAE,aAAa,EAAE,MAAM,YAAY,CAAC;AAEhD,OAAO,EAAoB,KAAK,cAAc,EAAE,MAAM,iBAAiB,CAAC;AA2BxE,MAAM,MAAM,kBAAkB,GAAG,cAAc,GAAG,YAAY,GAAG,WAAW,GAAG,OAAO,GAAG,WAAW,CAAC;AAErG,MAAM,WAAW,mBAAmB;IAClC,KAAK,EAAE,aAAa,CAAC;IACrB,MAAM,EAAE,aAAa,CAAC;IACtB,MAAM,EAAE,cAAc,CAAC;IACvB,UAAU,EAAE,MAAM,CAAC;IACnB,MAAM,CAAC,EAAE,MAAM,CAAC;CACjB;AAED,qBAAa,mBAAmB;IAkBlB,OAAO,CAAC,QAAQ,CAAC,IAAI;IAjBjC,OAAO,CAAC,UAAU,CAA8B;IAChD,OAAO,CAAC,iBAAiB,CAA+B;IACxD,OAAO,CAAC,eAAe,CAA+B;IACtD,OAAO,CAAC,cAAc,CAA+B;IACrD,OAAO,CAAC,qBAAqB,CAAS;IACtC,OAAO,CAAC,YAAY,CAAS;IAC7B,OAAO,CAAC,MAAM,CAAsC;IAEpD,OAAO,CAAC,iBAAiB,CAAS;IAElC,OAAO,CAAC,QAAQ,CAAC,GAAG,CAAS;IAC7B,OAAO,CAAC,QAAQ,CAAC,YAAY,CAAmB;IAChD,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAAgE;IACzF,OAAO,CAAC,QAAQ,CAAC,QAAQ,CAA0B;IACnD,OAAO,CAAC,QAAQ,CAAC,SAAS,CAA0B;IACpD,OAAO,CAAC,cAAc,CAAgB;gBAET,IAAI,EAAE,mBAAmB;IAgBtD,SAAS,IAAI,kBAAkB;IAI/B,cAAc,IAAI;QAAE,cAAc,EAAE,MAAM,CAAC;QAAC,WAAW,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAC;QAAC,SAAS,EAAE,MAAM,CAAA;KAAE;IAIvG,aAAa,IAAI;QAAE,EAAE,EAAE,cAAc,CAAC;QAAC,GAAG,EAAE,cAAc,CAAA;KAAE;IAI5D,iBAAiB,IAAI,MAAM,EAAE;IAIvB,cAAc,CAAC,QAAQ,EAAE,MAAM,GAAG,OAAO,CAAC,IAAI,CAAC;IAgB/C,OAAO,IAAI,OAAO,CAAC,IAAI,CAAC;IAsHxB,UAAU,IAAI,OAAO,CAAC,IAAI,CAAC;YAmBnB,gBAAgB;IA4B9B,OAAO,CAAC,iBAAiB;IAanB,mBAAmB,IAAI,OAAO,CAAC,IAAI,CAAC;YA2B5B,yBAAyB;YAgDzB,YAAY;IA0E1B,OAAO,CAAC,mBAAmB;IAM3B,OAAO,CAAC,gBAAgB;IAYxB,OAAO,CAAC,UAAU;YAeJ,IAAI;IASlB,OAAO,CAAC,SAAS;CAKlB"}

package/dist/core/orchestrator.js

@@ -1,5 +1,6 @@
 import { arch as osArch, platform as osPlatform } from "node:os";
 import { HttpTransportType, HubConnectionBuilder, LogLevel, } from "@microsoft/signalr";
+import { buildHostCapacity } from "./capacity.js";
 import { getLogger } from "./logger.js";
 import { listModelsSync } from "./models-fs.js";
 import { SharedGGUFRunner } from "./shared-runner.js";
@@ -32,6 +33,12 @@ export class OrchestratorService {
             paths: opts.paths,
             config: { schemaVersion: 1, lastWriter: "cli", schemaUpdatedAt: "" },
         });
+        this.sharedRunner.setWorkerCrashCallback((crash) => {
+            getLogger().error(`💥 llmWorker crash: model="${crash.model}" — clearing all active sessions`);
+            // All conversations are lost when the worker process dies — clear the session map
+            // so subsequent prompts trigger a fresh model load rather than using stale state.
+            this.sessions.clear();
+        });
     }
     getStatus() {
         return this.status;
@@ -257,6 +264,8 @@ export class OrchestratorService {
         const cfg = await this.opts.config.load();
         if (!cfg.hostId || !cfg.hostName)
             return;
+        const hostId = cfg.hostId;
+        const gpuIndex = cfg.selectedGpuIndex ?? 0;
         const capabilities = listModelsSync(this.opts.paths, cfg);
         try {
             await this.connection.invoke("RegisterHost", {
@@ -278,12 +287,25 @@ export class OrchestratorService {
         this.heartbeatInterval = setInterval(async () => {
             if (this.connection?.state !== "Connected")
                 return;
+            // v1 heartbeat — liveness only. Still required: the server uses it for
+            // uptime tracking and DB heartbeat persistence (HeartbeatV2 does neither).
             try {
-                await this.connection.send("Heartbeat", cfg.hostId);
+                await this.connection.send("Heartbeat", hostId);
             }
             catch (err) {
                 getLogger().warn("orchestrator: heartbeat error:", err.message);
             }
+            // v2 heartbeat — live capacity (GPU/CPU/VRAM/RAM/sessions). Lets the
+            // orchestrator route around hosts that are out of VRAM or overloaded
+            // instead of assigning prompts until the host OOMs and crashes. Kept
+            // separate from v1 so a metrics-sampling hiccup never drops liveness.
+            try {
+                const capacity = await buildHostCapacity(hostId, gpuIndex, this.sessions.size);
+                await this.connection.send("HeartbeatV2", capacity);
+            }
+            catch (err) {
+                getLogger().warn("orchestrator: heartbeatV2 error:", err.message);
+            }
         }, HEARTBEAT_INTERVAL_MS);
     }
     async handlePrompt(req) {