smolcoder-plus 1.0.0

package/dist/providers/lmstudio.js

@@ -0,0 +1,347 @@
+"use strict";
+// LM Studio adapter — standard OpenAI-compatible /v1/chat/completions with SSE
+// streaming. The context window is whatever LM Studio loaded the model with;
+// we detect it and budget within it (we cannot change it per request).
+//
+// Reasoning is the part that decides whether LM Studio feels fast or slow.
+// LM Studio's API accepts reasoning_effort none|minimal|low|medium|high|xhigh,
+// but each MODEL only supports a subset (read from /api/v1/models). A value
+// the model does not support is silently replaced by the model's DEFAULT —
+// which for current qwen3.x builds is "xhigh", the maximum. That is how a
+// harness asking for "high" ends up with 8,000-token thinking bursts per tool
+// call. So: "off" is sent as "none" (measured: fully disables thinking), and
+// every other level is snapped to the nearest level the model really has.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.LmStudioProvider = void 0;
+exports.toWire = toWire;
+exports.mapEffort = mapEffort;
+const types_1 = require("./types");
+const transport_1 = require("./transport");
+const scheduler_1 = require("./scheduler");
+const attachments_1 = require("../attachments");
+const util_1 = require("../util");
+/** Exported for tests. */
+function toWire(messages) {
+    return messages.map((m) => {
+        if (m.role === "assistant" && m.toolCalls?.length) {
+            return {
+                role: "assistant",
+                content: m.content || null,
+                tool_calls: m.toolCalls.map((tc) => ({
+                    id: tc.id,
+                    type: "function",
+                    function: { name: tc.name, arguments: JSON.stringify(tc.args) },
+                })),
+            };
+        }
+        if (m.role === "tool") {
+            return { role: "tool", tool_call_id: m.toolCallId, content: m.content };
+        }
+        if (m.role === "user" && m.images?.length) {
+            // OpenAI-style content parts: the text, then each image as a data URL.
+            const parts = [{ type: "text", text: m.content }];
+            for (const ref of m.images) {
+                const url = (0, attachments_1.imageDataUrl)(ref);
+                if (url)
+                    parts.push({ type: "image_url", image_url: { url } });
+            }
+            return { role: "user", content: parts.length > 1 ? parts : m.content };
+        }
+        return { role: m.role, content: m.content };
+    });
+}
+function toWireTools(tools) {
+    return tools.map((t) => ({
+        type: "function",
+        function: { name: t.name, description: t.description, parameters: t.parameters },
+    }));
+}
+/** Exported for tests. Map a smolcoder effort onto LM Studio's wire value,
+ * respecting what the model supports. Returns undefined for "leave it to the
+ * backend". */
+function mapEffort(effort, info) {
+    if (effort === null)
+        return undefined;
+    if (effort === "off")
+        return "none";
+    const wireLevels = ["low", "medium", "high", "xhigh"];
+    if (!info || info.allowed.length === 0)
+        return effort;
+    // Model-supported levels that the API also accepts (the model list uses
+    // "off"/"on" too; those are not valid wire values).
+    const candidates = wireLevels.filter((l) => info.allowed.includes(l));
+    if (candidates.length === 0)
+        return effort;
+    if (candidates.includes(effort))
+        return effort;
+    const want = types_1.EFFORT_RANK[effort];
+    let best = candidates[0];
+    let bestDist = Infinity;
+    for (const cnd of candidates) {
+        const d = Math.abs(types_1.EFFORT_RANK[cnd] - want);
+        // Ties go to the LOWER level: on a local model the cheaper step wins.
+        if (d < bestDist || (d === bestDist && types_1.EFFORT_RANK[cnd] < types_1.EFFORT_RANK[best])) {
+            best = cnd;
+            bestDist = d;
+        }
+    }
+    return best;
+}
+class LmStudioProvider {
+    baseUrl;
+    modelId;
+    contextWindow;
+    reasoning;
+    vision;
+    replaysThinking = false;
+    label;
+    maxOutputTokens;
+    effort = null;
+    effortUnsupported = false;
+    constructor(baseUrl, modelId, contextWindow, maxOutputTokens = types_1.MAX_OUTPUT_TOKENS, reasoning, 
+    /** Whether the model accepts images (LM Studio lists such models as "vlm"). */
+    vision) {
+        this.baseUrl = baseUrl;
+        this.modelId = modelId;
+        this.contextWindow = contextWindow;
+        this.reasoning = reasoning;
+        this.vision = vision;
+        this.label = `lmstudio · ${modelId}`;
+        this.maxOutputTokens = maxOutputTokens;
+    }
+    setEffort(effort) {
+        this.effort = effort;
+        this.effortUnsupported = false;
+    }
+    async loadedContextWindow() {
+        const data = await (0, util_1.tryFetchJson)(`${this.baseUrl}/api/v1/models`, undefined, 1500);
+        for (const model of data?.models ?? []) {
+            const instances = model.loaded_instances ?? [];
+            const instance = instances.find((m) => m.id === this.modelId) ?? (model.key === this.modelId ? instances[0] : undefined);
+            if (typeof instance?.config?.context_length === "number")
+                return instance.config.context_length;
+        }
+        return undefined;
+    }
+    effortLabel() {
+        if (this.effortUnsupported)
+            return this.effort ? `${this.effort} (ignored by this server)` : null;
+        if (this.effort === null) {
+            return this.reasoning?.default ? `default → ${this.reasoning.default}` : null;
+        }
+        const wire = mapEffort(this.effort, this.reasoning);
+        if (wire && wire !== this.effort && wire !== "none")
+            return `${this.effort} → ${wire}`;
+        return null;
+    }
+    async chat(messages, tools, opts = {}) {
+        return (0, scheduler_1.scheduleInference)(this.baseUrl, opts, (scheduled) => this.chatScheduled(messages, tools, scheduled));
+    }
+    async chatScheduled(messages, tools, opts) {
+        const effort = opts.effortOverride ?? this.effort;
+        let wireMessages = messages;
+        const base = {
+            model: this.modelId,
+            messages: toWire(wireMessages),
+            tools: tools.length ? toWireTools(tools) : undefined,
+            max_tokens: opts.maxTokens ?? this.maxOutputTokens,
+        };
+        const wireEffort = mapEffort(effort, this.reasoning);
+        if (wireEffort && !this.effortUnsupported) {
+            base.reasoning_effort = wireEffort;
+        }
+        else if (effort === "off" && /qwen/i.test(this.modelId)) {
+            // Older LM Studio builds without reasoning_effort: fall back to the
+            // qwen per-turn /no_think soft switch (older qwen3 models honor it).
+            wireMessages = messages.map((m) => ({ ...m }));
+            for (let i = wireMessages.length - 1; i >= 0; i--) {
+                if (wireMessages[i].role === "user") {
+                    wireMessages[i].content = wireMessages[i].content + " /no_think";
+                    break;
+                }
+            }
+            base.messages = toWire(wireMessages);
+        }
+        const started = { streaming: false };
+        try {
+            return await this.chain(base, opts, started);
+        }
+        catch (err) {
+            if (err?.name === "AbortError")
+                throw err;
+            if (started.streaming)
+                throw err; // tokens already shown — don't re-emit
+            const msg = String(err?.message ?? "");
+            // Only latch effortUnsupported on an actual param rejection (4xx naming
+            // it), never on a transient 5xx / dropped socket.
+            if (base.reasoning_effort && /returned 4\d\d/.test(msg) && /reasoning|effort/i.test(msg)) {
+                this.effortUnsupported = true;
+                delete base.reasoning_effort;
+                return await this.chain(base, opts, started);
+            }
+            throw err;
+        }
+    }
+    async chain(base, opts, started) {
+        try {
+            return await this.request({ ...base, stream: true, stream_options: { include_usage: true } }, opts, started);
+        }
+        catch (err) {
+            if (err?.name === "AbortError")
+                throw err;
+            if (started.streaming)
+                throw err;
+            const msg = String(err?.message ?? "");
+            // Only degrade the request shape on a pre-stream HTTP rejection; a
+            // transient error must propagate to chatWithRetry for backoff.
+            if (!/returned 4\d\d/.test(msg))
+                throw err;
+            // A reasoning_effort rejection must reach chat()'s handler, not be
+            // masked by the stream-shape fallback ladder.
+            if (/reasoning|effort/i.test(msg))
+                throw err;
+            try {
+                return await this.request({ ...base, stream: true }, opts, started);
+            }
+            catch (err2) {
+                if (err2?.name === "AbortError" || started.streaming)
+                    throw err2;
+                if (!/returned 4\d\d/.test(String(err2?.message ?? "")))
+                    throw err2;
+                return await this.request({ ...base, stream: false }, opts, started);
+            }
+        }
+    }
+    async request(body, opts, started) {
+        return (0, transport_1.withDeadline)(opts, (signal, activity) => this.readResponse(body, { ...opts, signal }, activity, started));
+    }
+    async readResponse(body, opts, activity, started) {
+        const t0 = Date.now();
+        const res = await fetch(`${this.baseUrl}/v1/chat/completions`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: opts.signal,
+        });
+        if (!res.ok) {
+            const text = await res.text().catch(() => "");
+            throw new Error(`LM Studio returned ${res.status}: ${text.slice(0, 300)}`);
+        }
+        if (body.stream === false) {
+            const data = await res.json();
+            if (data.error || !data.choices?.[0]?.message)
+                throw new Error(`LM Studio returned an invalid completion: ${data.error?.message ?? "missing message"}`);
+            const msg = data.choices?.[0]?.message ?? {};
+            const toolCalls = (msg.tool_calls ?? []).map((tc) => ({
+                id: tc.id || (0, types_1.nextCallId)(),
+                name: tc.function?.name ?? "",
+                ...(0, types_1.parseArgs)(tc.function?.arguments),
+            }));
+            if (started)
+                started.streaming = true;
+            if (msg.content)
+                opts.onToken?.(msg.content);
+            const content = msg.content ?? "";
+            const total = Date.now() - t0;
+            return {
+                content,
+                toolCalls,
+                promptTokens: data.usage?.prompt_tokens,
+                completionTokens: (0, types_1.estimateReplayTokens)(content, toolCalls),
+                generatedTokens: data.usage?.completion_tokens,
+                genTokPerSec: data.usage?.completion_tokens ? data.usage.completion_tokens / (total / 1000) : undefined,
+                truncated: data.choices?.[0]?.finish_reason === "length",
+            };
+        }
+        // SSE stream.
+        let content = "";
+        let thinking = "";
+        const partials = new Map();
+        let promptTokens;
+        let completionTokens;
+        let truncated = false;
+        let finished = false;
+        let firstTokAt = 0;
+        let lastTokAt = 0;
+        const handleLine = (rawLine) => {
+            const line = rawLine.trim();
+            if (!line.startsWith("data:"))
+                return;
+            const payload = line.slice(5).trim();
+            if (payload === "[DONE]")
+                return;
+            const chunk = (0, transport_1.streamJson)(payload);
+            if (started)
+                started.streaming = true; // committed — no safe re-request now
+            if (chunk.usage) {
+                promptTokens = chunk.usage.prompt_tokens ?? promptTokens;
+                completionTokens = chunk.usage.completion_tokens ?? completionTokens;
+            }
+            if (chunk.choices?.[0]?.finish_reason === "length")
+                truncated = true;
+            if (chunk.choices?.[0]?.finish_reason)
+                finished = true;
+            const delta = chunk.choices?.[0]?.delta;
+            if (!delta)
+                return;
+            const reasoning = delta.reasoning_content ?? delta.reasoning;
+            let sawToken = false;
+            if (typeof reasoning === "string" && reasoning) {
+                thinking += reasoning;
+                opts.onThinking?.(reasoning);
+                sawToken = true;
+            }
+            if (delta.content) {
+                content += delta.content;
+                opts.onToken?.(delta.content);
+                sawToken = true;
+            }
+            if (Array.isArray(delta.tool_calls)) {
+                sawToken = true;
+                for (const tc of delta.tool_calls) {
+                    const idx = tc.index ?? 0;
+                    const p = partials.get(idx) ?? { id: "", name: "", args: "" };
+                    if (tc.id)
+                        p.id = tc.id;
+                    if (tc.function?.name)
+                        p.name += tc.function.name;
+                    if (tc.function?.arguments)
+                        p.args += tc.function.arguments;
+                    partials.set(idx, p);
+                }
+            }
+            if (sawToken) {
+                const now = Date.now();
+                if (!firstTokAt)
+                    firstTokAt = now;
+                lastTokAt = now;
+            }
+        };
+        for await (const line of (0, transport_1.responseLines)(res, activity))
+            handleLine(line);
+        if (!finished)
+            throw new Error("LM Studio stream ended before completion; the response was discarded");
+        const toolCalls = [...partials.entries()]
+            .sort((a, b) => a[0] - b[0])
+            .map(([, p]) => ({
+            id: p.id || (0, types_1.nextCallId)(),
+            name: p.name,
+            ...(0, types_1.parseArgs)(p.args),
+        }));
+        const genMs = lastTokAt > firstTokAt ? lastTokAt - firstTokAt : 0;
+        return {
+            content,
+            toolCalls,
+            thinking: thinking || undefined,
+            promptTokens,
+            // LM Studio does not replay reasoning into the next prompt, so only the
+            // visible reply and tool-call JSON count toward the next request.
+            completionTokens: (0, types_1.estimateReplayTokens)(content, toolCalls),
+            generatedTokens: completionTokens,
+            genTokPerSec: completionTokens && genMs > 0 ? completionTokens / (genMs / 1000) : undefined,
+            ttftMs: firstTokAt ? firstTokAt - t0 : undefined,
+            truncated,
+        };
+    }
+}
+exports.LmStudioProvider = LmStudioProvider;

package/dist/providers/ollama.js

@@ -0,0 +1,269 @@
+"use strict";
+// Ollama adapter — uses the NATIVE /api/chat endpoint, not the OpenAI-compat
+// one, because only the native API lets us set num_ctx per request, pass
+// thinking traces back, and read real timings (prompt/eval durations).
+//
+// Prompt-size savings that matter on a long tool loop with a thinking model:
+// reasoning traces from assistant messages BEFORE the current user turn are
+// not sent back (the qwen3-family templates drop them anyway); only the
+// current turn's traces travel, which is what tool-call loops need.
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.OllamaProvider = void 0;
+exports.toWire = toWire;
+const types_1 = require("./types");
+const transport_1 = require("./transport");
+const scheduler_1 = require("./scheduler");
+const attachments_1 = require("../attachments");
+const util_1 = require("../util");
+/** Exported for tests. */
+function toWire(messages) {
+    const keepThinkingFrom = (0, types_1.lastUserIndex)(messages);
+    return messages.map((m, i) => {
+        if (m.role === "assistant" && m.toolCalls?.length) {
+            return {
+                role: "assistant",
+                content: m.content ?? "",
+                ...(m.thinking && i > keepThinkingFrom ? { thinking: m.thinking } : {}),
+                tool_calls: m.toolCalls.map((tc) => ({
+                    function: { name: tc.name, arguments: tc.args },
+                })),
+            };
+        }
+        if (m.role === "tool") {
+            return { role: "tool", content: m.content, tool_name: m.toolName };
+        }
+        if (m.role === "user" && m.images?.length) {
+            const images = m.images.map(attachments_1.imageBase64).filter((b) => b !== null);
+            return { role: "user", content: m.content, ...(images.length ? { images } : {}) };
+        }
+        return { role: m.role, content: m.content };
+    });
+}
+function toWireTools(tools) {
+    return tools.map((t) => ({
+        type: "function",
+        function: { name: t.name, description: t.description, parameters: t.parameters },
+    }));
+}
+/** Keep the model resident between tool calls and while the user reads or
+ * approves. Ollama's own default (5 min) unloads mid-session on any longer
+ * pause, and a reload of a 17 GB model costs 10-20 s plus a cold cache. */
+const KEEP_ALIVE = process.env.SMOLCODER_KEEP_ALIVE || "30m";
+class OllamaProvider {
+    baseUrl;
+    modelId;
+    contextWindow;
+    numCtx;
+    vision;
+    replaysThinking = true;
+    label;
+    maxOutputTokens;
+    effort = null;
+    thinkUnsupported = false;
+    /** null = unknown, tried lazily; false = this model only takes a boolean. */
+    levelsSupported = null;
+    constructor(baseUrl, modelId, contextWindow, 
+    /** Explicit num_ctx to send; undefined = respect the server's configured context. */
+    numCtx, maxOutputTokens = types_1.MAX_OUTPUT_TOKENS, 
+    /** Whether the model accepts images (from /api/show capabilities). */
+    vision) {
+        this.baseUrl = baseUrl;
+        this.modelId = modelId;
+        this.contextWindow = contextWindow;
+        this.numCtx = numCtx;
+        this.vision = vision;
+        this.label = `ollama · ${modelId}`;
+        this.maxOutputTokens = maxOutputTokens;
+        // Only gpt-oss is documented to take levels; everything else gets a
+        // boolean straight away instead of a wasted probe request.
+        if (!/gpt-oss/i.test(modelId))
+            this.levelsSupported = false;
+    }
+    setEffort(effort) {
+        this.effort = effort;
+        this.thinkUnsupported = false;
+    }
+    async loadedContextWindow() {
+        if (this.numCtx)
+            return this.numCtx;
+        const ps = await (0, util_1.tryFetchJson)(`${this.baseUrl}/api/ps`, undefined, 1500);
+        const model = ps?.models?.find((m) => m.name === this.modelId || m.model === this.modelId);
+        return typeof model?.context_length === "number" ? model.context_length : undefined;
+    }
+    effortLabel() {
+        if (this.effort === null || this.effort === "off")
+            return null;
+        if (this.thinkUnsupported)
+            return `${this.effort} (model has no thinking switch)`;
+        if (this.levelsSupported === false)
+            return `${this.effort} → thinking on`;
+        return null;
+    }
+    /** Ollama's think param: boolean for most reasoning models; gpt-oss accepts levels. */
+    thinkParam(effort) {
+        if (effort === null || this.thinkUnsupported)
+            return undefined;
+        if (effort === "off")
+            return false;
+        return this.levelsSupported === false ? true : effort;
+    }
+    async chat(messages, tools, opts = {}) {
+        return (0, scheduler_1.scheduleInference)(this.baseUrl, opts, (scheduled) => this.chatScheduled(messages, tools, scheduled));
+    }
+    async chatScheduled(messages, tools, opts) {
+        const effort = opts.effortOverride ?? this.effort;
+        const makeBody = (stream, think) => ({
+            model: this.modelId,
+            messages: toWire(messages),
+            tools: tools.length ? toWireTools(tools) : undefined,
+            stream,
+            keep_alive: KEEP_ALIVE,
+            ...(think !== undefined ? { think } : {}),
+            options: {
+                ...(this.numCtx ? { num_ctx: this.numCtx } : {}),
+                num_predict: opts.maxTokens ?? this.maxOutputTokens,
+            },
+        });
+        let think = this.thinkParam(effort);
+        const started = { streaming: false };
+        try {
+            return await this.request(makeBody(true, think), opts, started);
+        }
+        catch (err) {
+            if (err?.name === "AbortError")
+                throw err;
+            // Never re-request after tokens were already streamed to the UI — that
+            // double-emits. Let agent.ts's chatWithRetry handle mid-stream failures.
+            if (started.streaming)
+                throw err;
+            const msg = String(err?.message ?? "");
+            const paramRejected = /returned 4\d\d/.test(msg) && /think/i.test(msg);
+            // Only treat the think param as unsupported on an actual param rejection;
+            // a transient 5xx/network error must NOT permanently disable reasoning.
+            if (think !== undefined && paramRejected) {
+                if (typeof think === "string") {
+                    // Levels rejected — this model takes a boolean. Same intent: on.
+                    this.levelsSupported = false;
+                    think = true;
+                    try {
+                        return await this.request(makeBody(true, think), opts, started);
+                    }
+                    catch (err2) {
+                        if (err2?.name === "AbortError" || started.streaming)
+                            throw err2;
+                        const msg2 = String(err2?.message ?? "");
+                        if (!(/returned 4\d\d/.test(msg2) && /think/i.test(msg2)))
+                            throw err2;
+                    }
+                }
+                this.thinkUnsupported = true;
+                think = undefined;
+                return await this.request(makeBody(true, undefined), opts, started);
+            }
+            // Older Ollama versions reject stream+tools together; retry non-streaming
+            // once, but only for a pre-stream rejection (not a transient error).
+            if (/returned 4\d\d/.test(msg)) {
+                return await this.request(makeBody(false, think), opts, started);
+            }
+            throw err;
+        }
+    }
+    async request(body, opts, started) {
+        return (0, transport_1.withDeadline)(opts, (signal, activity) => this.readResponse(body, { ...opts, signal }, activity, started));
+    }
+    async readResponse(body, opts, activity, started) {
+        const t0 = Date.now();
+        const res = await fetch(`${this.baseUrl}/api/chat`, {
+            method: "POST",
+            headers: { "content-type": "application/json" },
+            body: JSON.stringify(body),
+            signal: opts.signal,
+        });
+        if (!res.ok) {
+            const text = await res.text().catch(() => "");
+            throw new Error(`Ollama returned ${res.status}: ${text.slice(0, 300)}`);
+        }
+        let content = "";
+        let thinking = "";
+        const toolCalls = [];
+        let promptTokens;
+        let completionTokens;
+        let promptTokPerSec;
+        let genTokPerSec;
+        let truncated = false;
+        let finished = false;
+        let firstTokAt = 0;
+        const handleChunk = (chunk) => {
+            if (chunk.error)
+                throw new Error(`Ollama stream error: ${chunk.error}`);
+            if (started)
+                started.streaming = true; // committed — no safe re-request now
+            const msg = chunk.message;
+            if (msg?.thinking) {
+                thinking += msg.thinking;
+                opts.onThinking?.(msg.thinking);
+                if (!firstTokAt)
+                    firstTokAt = Date.now();
+            }
+            if (msg?.content) {
+                content += msg.content;
+                opts.onToken?.(msg.content);
+                if (!firstTokAt)
+                    firstTokAt = Date.now();
+            }
+            if (Array.isArray(msg?.tool_calls)) {
+                if (!firstTokAt)
+                    firstTokAt = Date.now();
+                for (const tc of msg.tool_calls) {
+                    const fn = tc.function ?? {};
+                    toolCalls.push({
+                        id: (0, types_1.nextCallId)(),
+                        name: fn.name ?? "",
+                        ...(0, types_1.parseArgs)(fn.arguments),
+                    });
+                }
+            }
+            if (chunk.done) {
+                finished = true;
+                if (typeof chunk.prompt_eval_count === "number")
+                    promptTokens = chunk.prompt_eval_count;
+                if (typeof chunk.eval_count === "number")
+                    completionTokens = chunk.eval_count;
+                if (typeof chunk.prompt_eval_duration === "number" && chunk.prompt_eval_duration > 0 && promptTokens) {
+                    promptTokPerSec = promptTokens / (chunk.prompt_eval_duration / 1e9);
+                }
+                if (typeof chunk.eval_duration === "number" && chunk.eval_duration > 0 && completionTokens) {
+                    genTokPerSec = completionTokens / (chunk.eval_duration / 1e9);
+                }
+                if (chunk.done_reason === "length")
+                    truncated = true;
+            }
+        };
+        if (body.stream === false) {
+            handleChunk(await res.json());
+        }
+        else {
+            for await (const line of (0, transport_1.responseLines)(res, activity)) {
+                if (line)
+                    handleChunk((0, transport_1.streamJson)(line));
+            }
+        }
+        if (!finished)
+            throw new Error("Ollama stream ended before completion; the response was discarded");
+        return {
+            content,
+            toolCalls,
+            thinking: thinking || undefined,
+            promptTokens,
+            // Ollama replays this turn's thinking into the next prompt, so the full
+            // eval count is what the next request carries.
+            completionTokens,
+            generatedTokens: completionTokens,
+            promptTokPerSec,
+            genTokPerSec,
+            ttftMs: firstTokAt ? firstTokAt - t0 : undefined,
+            truncated,
+        };
+    }
+}
+exports.OllamaProvider = OllamaProvider;

package/dist/providers/scheduler.js

@@ -0,0 +1,57 @@
+"use strict";
+Object.defineProperty(exports, "__esModule", { value: true });
+exports.scheduleInference = scheduleInference;
+const servers = new Map();
+/** One inference per server. Coding preempts optional summaries and titles,
+ * avoiding a second model/KV allocation just to do maintenance. */
+function scheduleInference(server, opts, run) {
+    let state = servers.get(server);
+    if (!state) {
+        state = { queue: [] };
+        servers.set(server, state);
+    }
+    const queue = state;
+    if (opts.background && (queue.active || queue.queue.length))
+        return Promise.reject(new Error("Background inference deferred: server is busy"));
+    return new Promise((resolve, reject) => {
+        const controller = new AbortController();
+        const job = { background: !!opts.background, controller, start: () => { } };
+        const cancel = () => {
+            controller.abort(opts.signal?.reason);
+            const index = queue.queue.indexOf(job);
+            if (index >= 0) {
+                queue.queue.splice(index, 1);
+                opts.signal?.removeEventListener("abort", cancel);
+                reject(controller.signal.reason);
+            }
+        };
+        job.start = () => {
+            queue.active = job;
+            if (opts.signal?.aborted)
+                cancel();
+            Promise.resolve().then(() => {
+                if (controller.signal.aborted)
+                    throw controller.signal.reason;
+                return run({ ...opts, signal: controller.signal });
+            }).then(resolve, reject).finally(() => {
+                opts.signal?.removeEventListener("abort", cancel);
+                queue.active = undefined;
+                const next = queue.queue.shift();
+                if (next)
+                    next.start();
+                else
+                    servers.delete(server);
+            });
+        };
+        opts.signal?.addEventListener("abort", cancel, { once: true });
+        if (queue.active) {
+            queue.queue.push(job);
+            if (!job.background && queue.active.background)
+                queue.active.controller.abort();
+            if (opts.signal?.aborted)
+                cancel();
+        }
+        else
+            job.start();
+    });
+}