sublera-cli 0.0.0-stage → 1.3.0

package/dist/engine/llamaRunner.js

@@ -0,0 +1,282 @@
+import { spawn } from 'node:child_process';
+import { EventEmitter } from 'node:events';
+import { createWriteStream } from 'node:fs';
+import { mkdir } from 'node:fs/promises';
+import { createServer } from 'node:net';
+import { join } from 'node:path';
+import { logsDir } from '../config.js';
+import { killTree, sleep } from '../util/exec.js';
+export function buildLlamaArgs(options) {
+    const slots = Math.max(1, Math.round(options.slots ?? 1));
+    // llama-server treats --ctx-size as the total KV cache and divides it across
+    // parallel slots. Sublera advertises the per-slot context, so allocate slots * ctx.
+    const totalContext = Math.max(256, Math.round(options.context)) * slots;
+    const args = [
+        '-m', options.modelPath,
+        '--host', options.host || '127.0.0.1',
+        '--port', String(options.port || 0),
+        '--n-gpu-layers', String(options.gpuLayers ?? 999),
+        '--ctx-size', String(totalContext),
+        '--parallel', String(slots),
+        '--cont-batching',
+        '--log-disable',
+    ];
+    if (options.extraArgs?.length)
+        args.push(...options.extraArgs);
+    return args;
+}
+/**
+ * llama-server reads its own settings from the environment: LLAMA_ARG_* (any flag, such as a
+ * chat template, a KV-cache type, or sampling defaults), LLAMA_LOG_* and LLAMA_API_KEY. A host
+ * who also uses llama.cpp privately may have these set. They must not reach the engine: buyers
+ * were promised the locked flags, the server's model checks expect them, an API key would lock
+ * the agent out of its own engine, and a log setting could write prompts to disk.
+ */
+export function engineEnv(base, extra = {}) {
+    const env = {};
+    const dropped = [];
+    // Overrides are no more trusted than the inherited environment. Filter after merging so
+    // an explicit env option cannot reintroduce authentication, logging, or generation flags.
+    for (const [name, value] of Object.entries({ ...base, ...extra })) {
+        if (/^(LLAMA_ARG_|LLAMA_LOG_)/i.test(name) || /^LLAMA_API_KEY$/i.test(name))
+            dropped.push(name);
+        else
+            env[name] = value;
+    }
+    return { env, dropped: dropped.sort() };
+}
+export async function findFreePort(host = '127.0.0.1') {
+    return new Promise((resolve, reject) => {
+        const server = createServer();
+        server.unref();
+        server.on('error', reject);
+        server.listen(0, host, () => {
+            const address = server.address();
+            const port = typeof address === 'object' && address ? address.port : 0;
+            server.close(() => port ? resolve(port) : reject(new Error('Could not allocate a local port')));
+        });
+    });
+}
+export class LlamaRunner extends EventEmitter {
+    options;
+    logger;
+    child = null;
+    logStream = null;
+    _status = 'stopped';
+    _port = 0;
+    _url = '';
+    _props = null;
+    _pid = null;
+    stopping = false;
+    restartTimes = [];
+    startPromise = null;
+    restartTimer = null;
+    _restarts = 0;
+    failuresSinceReady = 0;
+    constructor(options) {
+        super();
+        this.options = options;
+        this.logger = options.logger;
+    }
+    get status() { return this._status; }
+    get port() { return this._port; }
+    get url() { return this._url; }
+    get pid() { return this._pid; }
+    get props() { return this._props; }
+    get restarts() { return this._restarts; }
+    setStatus(status) {
+        if (this._status === status)
+            return;
+        this._status = status;
+        this.emit('status', status);
+    }
+    async start() {
+        if (this._status === 'ready' && this._props)
+            return this._props;
+        if (this.startPromise)
+            return this.startPromise;
+        this.startPromise = this.startInternal();
+        try {
+            return await this.startPromise;
+        }
+        finally {
+            this.startPromise = null;
+        }
+    }
+    async startInternal() {
+        this.stopping = false;
+        this.setStatus(this._restarts > 0 ? 'restarting' : 'starting');
+        const port = this.options.port ?? await findFreePort(this.options.host || '127.0.0.1');
+        this._port = port;
+        this._url = `http://${this.options.host || '127.0.0.1'}:${port}`;
+        const args = [
+            ...(this.options.baseArgs || []),
+            ...buildLlamaArgs({ ...this.options, port }),
+        ];
+        this.logger?.info('Starting llama-server', { port, context: this.options.context, slots: this.options.slots ?? 1 });
+        await mkdir(logsDir(), { recursive: true }).catch(() => undefined);
+        if (this.stopping)
+            throw new Error('llama-server start was cancelled');
+        const logFile = this.options.logFile ?? join(logsDir(), 'llama-server.log');
+        this.logStream?.end();
+        this.logStream = createWriteStream(logFile, { flags: 'a' });
+        const { env, dropped } = engineEnv(process.env, this.options.env);
+        if (dropped.length && this._restarts === 0) {
+            this.logger?.warn(`Ignoring your llama.cpp settings for this engine: ${dropped.join(', ')}. Sublera runs llama-server with fixed flags.`);
+        }
+        const child = spawn(this.options.binaryPath, args, {
+            env,
+            windowsHide: true,
+            stdio: ['ignore', 'pipe', 'pipe'],
+        });
+        this.child = child;
+        this._pid = child.pid ?? null;
+        const handleLine = (line) => {
+            this.options.onLog?.(line);
+            this.logStream?.write(`${new Date().toISOString()} ${line}\n`);
+        };
+        child.stdout?.on('data', (chunk) => {
+            for (const line of chunk.toString('utf8').split(/\r?\n/))
+                if (line.trim())
+                    handleLine(line);
+        });
+        child.stderr?.on('data', (chunk) => {
+            for (const line of chunk.toString('utf8').split(/\r?\n/))
+                if (line.trim())
+                    handleLine(line);
+        });
+        child.once('error', (error) => {
+            handleLine(`spawn error: ${error.message}`);
+            this.setStatus('failed');
+            this.emit('fatal', error);
+        });
+        let ready = false;
+        child.once('exit', (code, signal) => {
+            const wasStopping = this.stopping;
+            this.child = null;
+            this._pid = null;
+            this._props = null;
+            if (wasStopping) {
+                this.setStatus('stopped');
+                return;
+            }
+            handleLine(`llama-server exited (${code ?? signal ?? 'unknown'})`);
+            if (!ready)
+                return;
+            this.emit('crash', { code, signal });
+            this.scheduleRestart();
+        });
+        try {
+            const props = await this.waitForHealth(child);
+            this._props = props;
+            const seen = Number(props.n_ctx || props.default_generation_settings?.n_ctx || 0);
+            if (seen > 0 && seen < this.options.context) {
+                throw new Error(`llama-server loaded context ${seen}, below the locked ${this.options.context}`);
+            }
+            ready = true;
+            this.failuresSinceReady = 0;
+            this.setStatus('ready');
+            this.emit('ready', props);
+            return props;
+        }
+        catch (error) {
+            this.setStatus(this.stopping ? 'stopped' : 'failed');
+            child.removeAllListeners('exit');
+            this.child = null;
+            this._pid = null;
+            await killTree(child);
+            throw error;
+        }
+    }
+    async waitForHealth(child) {
+        const timeoutMs = this.options.healthTimeoutMs ?? 180_000;
+        const started = Date.now();
+        let lastError = '';
+        while (Date.now() - started < timeoutMs) {
+            if (this.stopping)
+                throw new Error('llama-server start was cancelled');
+            if (child.exitCode != null || !this.child)
+                throw new Error(`llama-server exited during startup (${child.exitCode ?? 'signal'})`);
+            try {
+                const health = await fetch(`${this._url}/health`, { signal: AbortSignal.timeout(2_000) });
+                if (health.ok) {
+                    const props = await fetch(`${this._url}/props`, { signal: AbortSignal.timeout(3_000) }).catch(() => null);
+                    if (props?.ok)
+                        return await props.json();
+                    return {};
+                }
+            }
+            catch (error) {
+                lastError = error instanceof Error ? error.message : String(error);
+            }
+            await sleep(500);
+        }
+        throw new Error(`llama-server did not become healthy within ${Math.round(timeoutMs / 1000)}s${lastError ? ` (${lastError})` : ''}`);
+    }
+    scheduleRestart() {
+        if (this.stopping)
+            return;
+        const restart = this.options.restart;
+        if (restart === false) {
+            this.setStatus('failed');
+            return;
+        }
+        const maxRestarts = restart?.maxRestarts ?? 5;
+        const windowMs = restart?.windowMs ?? 120_000;
+        const now = Date.now();
+        this.restartTimes = this.restartTimes.filter((t) => now - t < windowMs);
+        const reason = this.failuresSinceReady >= maxRestarts
+            ? `llama-server did not become ready after ${this.failuresSinceReady} restarts`
+            : this.restartTimes.length >= maxRestarts
+                ? `llama-server crashed ${this.restartTimes.length} times in ${Math.round(windowMs / 1000)}s`
+                : null;
+        if (reason) {
+            this.setStatus('failed');
+            this.emit('fatal', new Error(reason));
+            return;
+        }
+        this.restartTimes.push(now);
+        this.failuresSinceReady++;
+        this._restarts++;
+        this.setStatus('restarting');
+        const delay = restart?.delayMs ?? 500;
+        if (this.restartTimer)
+            clearTimeout(this.restartTimer);
+        this.restartTimer = setTimeout(() => {
+            this.restartTimer = null;
+            if (this.stopping)
+                return;
+            void this.start().catch((error) => {
+                this.logger?.error('llama-server restart failed', { error: error instanceof Error ? error.message : String(error) });
+                this.scheduleRestart();
+            });
+        }, delay);
+    }
+    async stop() {
+        this.stopping = true;
+        if (this.restartTimer) {
+            clearTimeout(this.restartTimer);
+            this.restartTimer = null;
+        }
+        const child = this.child;
+        this.child = null;
+        this._props = null;
+        if (child)
+            await killTree(child);
+        this.logStream?.end();
+        this.logStream = null;
+        this.setStatus('stopped');
+    }
+    async isHealthy() {
+        if (!this._url || this._status !== 'ready')
+            return false;
+        try {
+            const res = await fetch(`${this._url}/health`, { signal: AbortSignal.timeout(2_000) });
+            return res.ok;
+        }
+        catch {
+            return false;
+        }
+    }
+}
+//# sourceMappingURL=llamaRunner.js.map

package/dist/engine/modelManager.d.ts

@@ -0,0 +1,63 @@
+import { type GgufInfo } from './gguf.js';
+import type { CachedModelInfo } from '../types.js';
+export type ModelProgress = {
+    phase: 'download' | 'verify' | 'header' | 'cache';
+    received: number;
+    total: number | null;
+    percent: number | null;
+    speedBps: number | null;
+    etaSeconds: number | null;
+    note: string;
+};
+export type LoadedModel = {
+    path: string;
+    file: string;
+    url: string;
+    sha256: string;
+    size: number;
+    info: GgufInfo;
+    paramB: number | null;
+    quant: string | null;
+    layers: number | null;
+    contextTrain: number | null;
+    architecture: string | null;
+    needsMb: number | null;
+    /** A supplied expected SHA-256 matched measured file bytes; a computed digest alone is not verification. */
+    verified: boolean;
+};
+export type DownloadModelOptions = {
+    url: string;
+    expectedSha256?: string | null;
+    expectedSize?: number | null;
+    expectedLayers?: number | null;
+    context?: number;
+    cacheDir?: string;
+    onProgress?: (progress: ModelProgress) => void;
+    signal?: AbortSignal;
+    fetchImpl?: typeof fetch;
+    showProgress?: boolean;
+    stream?: NodeJS.WriteStream;
+    slots?: number;
+    /** Unit tests only: allow http(s) downloads from a loopback address. */
+    allowLocalhost?: boolean;
+    /**
+     * Retained for caller compatibility. Cached files are always hashed before reporting their
+     * SHA-256, unless this process already hashed the same file identity and change timestamps.
+     */
+    verifyCached?: boolean;
+};
+/** Unit tests: behave like a new agent process that has hashed nothing yet. */
+export declare function forgetHashedModels(): void;
+export declare function modelPathFor(url: string, cacheDir?: string): string;
+/** Cache inventory hints, not proof of a hash check. downloadModel revalidates bytes before use. */
+export declare function listCachedModels(cacheDir?: string): Promise<CachedModelInfo[]>;
+/** Serialize downloads so concurrent jobs in this process cannot spend the same disk budget. */
+export declare function downloadModel(options: DownloadModelOptions): Promise<LoadedModel>;
+export declare function verifyModelFile(path: string, expected?: {
+    sha256?: string | null;
+    layers?: number | null;
+    context?: number;
+    slots?: number;
+}): Promise<LoadedModel>;
+export declare function cachedModelName(url: string): string;
+export declare function openModelHeader(path: string, bytes?: number): Promise<GgufInfo>;