pi-nebius 0.3.0

package/dist/discovery.js

@@ -0,0 +1,154 @@
+import { createHash, randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, unlink, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { BASE_URL, isRecord, parseModels } from "./models.js";
+export const MISSING_KEY = 'Nebius Token Factory: set export NEBIUS_API_KEY="..." and restart Pi.';
+const TTL_MS = 24 * 60 * 60 * 1000;
+const MAX_BYTES = 4 * 1024 * 1024;
+export class DiscoveryError extends Error {
+    status;
+    constructor(message, status) {
+        super(message);
+        this.status = status;
+    }
+}
+export function cachePath(agentDir, apiKey) {
+    const fingerprint = createHash("sha256").update(`${BASE_URL}\0${apiKey}`).digest("hex");
+    return join(agentDir, "cache", "pi-nebius", `${fingerprint}.json`);
+}
+async function readCache(path) {
+    try {
+        const raw = await readFile(path, "utf8");
+        if (Buffer.byteLength(raw) > MAX_BYTES)
+            return undefined;
+        const value = JSON.parse(raw);
+        if (!isRecord(value) ||
+            value.version !== 1 ||
+            typeof value.checkedAt !== "number" ||
+            !Number.isFinite(value.checkedAt) ||
+            !Array.isArray(value.models))
+            return undefined;
+        // Cache only normalized discovery fields, and remap through the same validation path.
+        return {
+            checkedAt: value.checkedAt,
+            models: parseModels({ object: "list", data: value.models }),
+        };
+    }
+    catch {
+        return undefined;
+    }
+}
+async function writeCache(path, models, checkedAt) {
+    const temp = `${path}.${randomUUID()}.tmp`;
+    try {
+        await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+        await writeFile(temp, JSON.stringify({
+            version: 1,
+            checkedAt,
+            models: models.map((model) => ({
+                id: model.id,
+                name: model.name,
+                context_length: model.contextWindow,
+                architecture: {
+                    modality: model.input.includes("image") ? "text+image->text" : "text->text",
+                },
+                supported_features: model.reasoning ? ["reasoning"] : [],
+                supported_sampling_parameters: model.nebiusSupportedParameters ?? [],
+            })),
+        }), { mode: 0o600, flag: "wx" });
+        await rename(temp, path);
+    }
+    finally {
+        await unlink(temp).catch(() => { });
+    }
+}
+export async function discoverModels(options) {
+    const key = options.apiKey?.trim();
+    if (!key)
+        return { models: [], source: "none", warning: MISSING_KEY };
+    const path = cachePath(options.agentDir, key);
+    const cached = await readCache(path);
+    const now = options.now ?? Date.now();
+    if (!options.force && cached && now >= cached.checkedAt && now - cached.checkedAt < TTL_MS) {
+        return { models: cached.models, source: "cache" };
+    }
+    try {
+        const signal = AbortSignal.any([
+            AbortSignal.timeout(8000),
+            ...(options.signal ? [options.signal] : []),
+        ]);
+        const response = await (options.fetch ?? globalThis.fetch)(`${BASE_URL}/models?verbose=true`, {
+            headers: { Authorization: `Bearer ${key}`, Accept: "application/json" },
+            signal,
+            redirect: "error",
+        });
+        if (!response.ok) {
+            await response.body?.cancel();
+            const detail = response.status === 401 || response.status === 403
+                ? "Token Factory rejected the credentials; check NEBIUS_API_KEY and project access."
+                : response.status === 429
+                    ? "Token Factory rate limit reached."
+                    : response.status >= 500
+                        ? "Token Factory is temporarily unavailable."
+                        : "Token Factory model discovery failed.";
+            const retry = response.headers.get("retry-after");
+            // Do not log arbitrary error bodies: they may echo credentials/request headers.
+            const safeRetry = retry && /^[\w ,:+.-]{1,100}$/.test(retry) ? ` Retry-After: ${retry}.` : "";
+            throw new DiscoveryError(`HTTP ${response.status}: ${detail}${safeRetry}`, response.status);
+        }
+        const reader = response.body?.getReader();
+        if (!reader)
+            throw new DiscoveryError("Empty Token Factory model response.");
+        const chunks = [];
+        let size = 0;
+        try {
+            while (true) {
+                const { done, value } = await reader.read();
+                if (done)
+                    break;
+                size += value.byteLength;
+                if (size > MAX_BYTES)
+                    throw new DiscoveryError("Token Factory model response exceeds 4 MiB.");
+                chunks.push(value);
+            }
+        }
+        finally {
+            await reader.cancel().catch(() => { });
+        }
+        let payload;
+        try {
+            payload = JSON.parse(Buffer.concat(chunks).toString("utf8"));
+        }
+        catch {
+            throw new DiscoveryError("Malformed JSON in Token Factory model response.");
+        }
+        const models = parseModels(payload);
+        try {
+            await writeCache(path, models, now);
+        }
+        catch {
+            return {
+                models,
+                source: "network",
+                warning: "Nebius models loaded, but the metadata cache could not be written.",
+            };
+        }
+        return { models, source: "network" };
+    }
+    catch (error) {
+        const authFailure = error instanceof DiscoveryError && (error.status === 401 || error.status === 403);
+        if (authFailure)
+            await unlink(path).catch(() => { });
+        const fallback = !authFailure && !options.signal?.aborted ? cached : undefined;
+        const detail = error instanceof DiscoveryError
+            ? error.message
+            : error instanceof Error && error.message.startsWith("Malformed Token Factory")
+                ? error.message
+                : "Token Factory model discovery could not complete (connectivity, timeout, or cancellation).";
+        return {
+            models: fallback?.models ?? [],
+            source: fallback ? "stale" : "none",
+            warning: `Nebius: ${detail} ${fallback ? "Using stale cached models." : "Use models.json for an offline model definition, then retry with /nebius-refresh."}`,
+        };
+    }
+}

package/dist/errors.js

@@ -0,0 +1,32 @@
+/** Preserve retry diagnostics that Pi's SDK error-to-string conversion otherwise drops. */
+export function withErrorDetails(fetcher) {
+    return async (input, init) => {
+        const response = await fetcher(input, init);
+        if (response.ok)
+            return response; // Successful SSE bodies remain completely untouched.
+        const details = [];
+        for (const [name, value] of response.headers) {
+            if (name === "retry-after" ||
+                name === "retry-after-ms" ||
+                name === "x-request-id" ||
+                name.startsWith("x-ratelimit-") ||
+                name.startsWith("ratelimit-")) {
+                details.push(`${name}: ${value}`);
+            }
+        }
+        if (details.length === 0)
+            return response;
+        const body = await response.text();
+        const headers = new Headers(response.headers);
+        headers.delete("content-length");
+        headers.delete("content-encoding");
+        headers.set("content-type", "text/plain; charset=utf-8");
+        // Headers come first so Pi's error-body truncation keeps retry diagnostics.
+        // The original body is retained verbatim, including context-overflow markers.
+        return new Response(`${details.join("\n")}\n${body}`, {
+            status: response.status,
+            statusText: response.statusText,
+            headers,
+        });
+    };
+}

package/dist/index.js

@@ -0,0 +1,86 @@
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { registerBenchmarkCommand } from "./benchmark/command.js";
+import { discoverModels, MISSING_KEY } from "./discovery.js";
+import { applyModelSettings, loadSettings, settingsPath, } from "./model-settings.js";
+import { registerModelSettingsCommand } from "./model-settings-command.js";
+import { nebiusProvider } from "./provider.js";
+export default async function nebius(pi) {
+    const path = settingsPath(getAgentDir());
+    let settings = {};
+    try {
+        settings = await loadSettings(path);
+    }
+    catch (error) {
+        process.stderr.write(`Nebius settings were not loaded: ${String(error)}\n`);
+    }
+    let catalog = [];
+    const register = () => pi.registerProvider(nebiusProvider(catalog.map((model) => applyModelSettings(model, settings[model.id])), settings));
+    registerBenchmarkCommand(pi, () => settings);
+    registerModelSettingsCommand(pi, {
+        path,
+        models: () => catalog,
+        settings: () => settings,
+        update: (updated) => {
+            settings = updated;
+            register();
+        },
+    });
+    let pending;
+    const initialize = (force = false) => {
+        pending ??= discoverModels({
+            apiKey: process.env.NEBIUS_API_KEY,
+            agentDir: getAgentDir(),
+            force,
+        })
+            .then((result) => {
+            catalog = result.models;
+            register();
+            return result;
+        })
+            .finally(() => {
+            pending = undefined;
+        });
+        return pending;
+    };
+    const initial = await initialize();
+    if (initial.warning)
+        process.stderr.write(`${initial.warning}\n`);
+    pi.on("session_start", async (_event, ctx) => {
+        if (initial.warning && ctx.hasUI)
+            ctx.ui.notify(initial.warning, "warning");
+    });
+    pi.registerCommand("nebius-refresh", {
+        description: "Refresh the Nebius Token Factory model catalog",
+        handler: async (_args, ctx) => {
+            const result = await initialize(true);
+            const message = result.warning ?? `Nebius: loaded ${result.models.length} models.`;
+            if (ctx.hasUI)
+                ctx.ui.notify(message, result.warning ? "warning" : "info");
+            else
+                process.stderr.write(`${message}\n`);
+        },
+    });
+    pi.on("message_end", async (event) => {
+        const message = event.message;
+        if (message.role !== "assistant" ||
+            message.provider !== "nebius" ||
+            message.stopReason !== "error")
+            return;
+        const original = message.errorMessage ?? "Unknown provider error";
+        if (original.startsWith("Nebius Token Factory:"))
+            return;
+        const hint = /\b(?:401|403)\b/.test(original)
+            ? "Token Factory rejected the credentials; check NEBIUS_API_KEY and project access."
+            : /\b429\b|rate.?limit/i.test(original)
+                ? "Rate limited; retry after the server's requested delay."
+                : /\b404\b|model.*(?:not found|unavailable|does not exist)/i.test(original)
+                    ? "Model unavailable; run /nebius-refresh and select a current model with /model."
+                    : /\b5\d\d\b/.test(original)
+                        ? "Token Factory is temporarily unavailable; retry later."
+                        : /no.*(?:api.?key|auth)|not configured/i.test(original)
+                            ? MISSING_KEY
+                            : "Request failed.";
+        // Preserve Pi's normalized error, including status, retry hints, and overflow markers.
+        return { message: { ...message, errorMessage: `Nebius Token Factory: ${hint}\n${original}` } };
+    });
+}

package/dist/model-settings-command.js

@@ -0,0 +1,130 @@
+import { applyModelSettings, efforts, saveModelSettings, validateSettings, } from "./model-settings.js";
+export function registerModelSettingsCommand(pi, options) {
+    pi.registerCommand("nebius-model", {
+        description: "Configure saved per-model Nebius settings",
+        handler: async (args, ctx) => {
+            if (!ctx.hasUI) {
+                ctx.ui.notify("/nebius-model requires interactive Pi.", "warning");
+                return;
+            }
+            if (!ctx.isIdle()) {
+                ctx.ui.notify("Wait for the current response before changing model settings.", "warning");
+                return;
+            }
+            const models = options.models();
+            const id = args.trim() ||
+                (await ctx.ui.select("Nebius model settings", models.map((model) => model.id)));
+            if (!id)
+                return;
+            const base = models.find((model) => model.id === id);
+            if (!base) {
+                ctx.ui.notify("Unknown model. Run /nebius-refresh first.", "error");
+                return;
+            }
+            try {
+                while (true) {
+                    const current = options.settings()[id] ?? {};
+                    const effective = applyModelSettings(base, current);
+                    const choices = [
+                        `Temperature: ${current.temperature ?? "inherit"}`,
+                        `Reasoning effort: ${current.reasoningEffort ?? "inherit"}`,
+                        `Maximum output tokens: ${current.maxTokens ?? `inherit (${base.maxTokens})`}`,
+                        "Advanced metadata settings",
+                        "Reset all overrides",
+                        "Done",
+                    ];
+                    const action = await ctx.ui.select(id, choices);
+                    if (!action || action === "Done")
+                        return;
+                    const next = { ...current };
+                    if (action === "Reset all overrides") {
+                        for (const key of Object.keys(next))
+                            delete next[key];
+                    }
+                    else if (action === "Advanced metadata settings") {
+                        const advanced = await ctx.ui.select("Local metadata only; does not change server capabilities", ["Context window", "Reasoning support", "Back"]);
+                        if (!advanced || advanced === "Back")
+                            continue;
+                        if (advanced === "Context window") {
+                            const value = await ctx.ui.input("Context window: positive integer; blank to inherit", String(current.contextWindow ?? ""));
+                            if (value === undefined)
+                                continue;
+                            if (!value.trim())
+                                delete next.contextWindow;
+                            else
+                                next.contextWindow = Number(value);
+                        }
+                        else {
+                            const value = await ctx.ui.select("Reasoning support", [
+                                "Inherit",
+                                "Enabled",
+                                "Disabled",
+                            ]);
+                            if (!value)
+                                continue;
+                            if (value === "Inherit")
+                                delete next.reasoning;
+                            else
+                                next.reasoning = value === "Enabled";
+                            if (value === "Disabled")
+                                delete next.reasoningEffort;
+                        }
+                    }
+                    else if (action === choices[1]) {
+                        if (!effective.reasoning || !effective.compat?.supportsReasoningEffort) {
+                            ctx.ui.notify("The catalog does not advertise reasoning-effort control for this model. No override will be sent.", "warning");
+                            continue;
+                        }
+                        const value = await ctx.ui.select("Reasoning effort (saved value overrides Pi's thinking setting)", ["Inherit", ...efforts]);
+                        if (!value)
+                            continue;
+                        if (value === "Inherit")
+                            delete next.reasoningEffort;
+                        else
+                            next.reasoningEffort = value;
+                    }
+                    else {
+                        const temperature = action === choices[0];
+                        if (temperature && !base.nebiusSupportedParameters?.includes("temperature")) {
+                            ctx.ui.notify("The catalog does not advertise temperature control for this model. No override will be sent.", "warning");
+                            continue;
+                        }
+                        const key = temperature ? "temperature" : "maxTokens";
+                        const value = await ctx.ui.input(temperature
+                            ? "Temperature: 0–2; blank to inherit"
+                            : "Maximum output tokens: positive integer; blank to inherit", String(current[key] ?? ""));
+                        if (value === undefined)
+                            continue;
+                        if (!value.trim())
+                            delete next[key];
+                        else
+                            next[key] = Number(value);
+                    }
+                    try {
+                        validateSettings(next);
+                        if (next.maxTokens !== undefined &&
+                            next.maxTokens > (next.contextWindow ?? base.contextWindow))
+                            throw new Error("Maximum output tokens cannot exceed the context window.");
+                    }
+                    catch (error) {
+                        ctx.ui.notify(String(error), "error");
+                        continue;
+                    }
+                    const saved = await saveModelSettings(options.path, id, next);
+                    options.update(saved);
+                    if (ctx.model?.provider === "nebius" && ctx.model.id === id) {
+                        const model = ctx.modelRegistry
+                            .getAll()
+                            .find((candidate) => candidate.provider === "nebius" && candidate.id === id);
+                        if (model)
+                            await pi.setModel(model);
+                    }
+                    ctx.ui.notify("Saved. New requests use these settings; running benchmarks keep their starting settings.", "info");
+                }
+            }
+            catch (error) {
+                ctx.ui.notify(`Could not update Nebius settings: ${String(error)}`, "error");
+            }
+        },
+    });
+}

package/dist/model-settings.js

@@ -0,0 +1,101 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, readFile, rename, rm, writeFile } from "node:fs/promises";
+import { dirname, join } from "node:path";
+import { isRecord } from "./models.js";
+export const efforts = ["none", "minimal", "low", "medium", "high", "xhigh", "max"];
+export const settingsPath = (agentDir) => join(agentDir, "pi-nebius", "model-settings.json");
+export function validateSettings(value) {
+    if (!isRecord(value))
+        throw new Error("Model settings must be an object.");
+    const allowed = ["temperature", "reasoningEffort", "maxTokens", "contextWindow", "reasoning"];
+    for (const key of Object.keys(value)) {
+        if (!allowed.includes(key))
+            throw new Error(`Unknown model setting: ${key}`);
+        const v = value[key];
+        if (key === "temperature" && (typeof v !== "number" || !Number.isFinite(v) || v < 0 || v > 2))
+            throw new Error("Temperature must be between 0 and 2.");
+        if ((key === "maxTokens" || key === "contextWindow") &&
+            (typeof v !== "number" || !Number.isSafeInteger(v) || v <= 0))
+            throw new Error(`${key} must be a positive integer.`);
+        if (key === "reasoningEffort" && !efforts.includes(v))
+            throw new Error("Invalid reasoning effort.");
+        if (key === "reasoning" && typeof v !== "boolean")
+            throw new Error("Reasoning must be a boolean.");
+    }
+    if (typeof value.maxTokens === "number" &&
+        typeof value.contextWindow === "number" &&
+        value.maxTokens > value.contextWindow)
+        throw new Error("Maximum output tokens cannot exceed the context window.");
+    return { ...value };
+}
+export async function loadSettings(path) {
+    let text;
+    try {
+        text = await readFile(path, "utf8");
+    }
+    catch (error) {
+        if (error.code === "ENOENT")
+            return {};
+        throw error;
+    }
+    const value = JSON.parse(text);
+    if (!isRecord(value) || value.version !== 1 || !isRecord(value.models))
+        throw new Error("Invalid Nebius model settings file.");
+    return Object.fromEntries(Object.entries(value.models).map(([id, settings]) => [id, validateSettings(settings)]));
+}
+export async function saveModelSettings(path, id, settings) {
+    // Re-read before each edit so unrelated changes from another Pi session are retained.
+    const models = await loadSettings(path);
+    const next = validateSettings(settings);
+    const entries = Object.entries(models).filter(([key]) => key !== id);
+    if (Object.keys(next).length)
+        entries.push([id, next]);
+    const updated = Object.fromEntries(entries);
+    await mkdir(dirname(path), { recursive: true, mode: 0o700 });
+    const temporary = `${path}.${randomUUID()}.tmp`;
+    try {
+        await writeFile(temporary, `${JSON.stringify({ version: 1, models: updated }, null, 2)}\n`, {
+            mode: 0o600,
+            flag: "wx",
+        });
+        await rename(temporary, path);
+    }
+    finally {
+        await rm(temporary, { force: true });
+    }
+    return updated;
+}
+export function applyModelSettings(model, settings = {}) {
+    const contextWindow = settings.contextWindow ?? model.contextWindow;
+    return {
+        ...model,
+        contextWindow,
+        maxTokens: Math.min(settings.maxTokens ?? model.maxTokens, contextWindow),
+        reasoning: settings.reasoning ?? model.reasoning,
+    };
+}
+/** Apply only supported request overrides; never store prompts or messages. */
+export function applyRequestSettings(payload, model, settings = {}) {
+    if (!isRecord(payload))
+        return payload;
+    const result = { ...payload };
+    if (settings.temperature !== undefined &&
+        model.nebiusSupportedParameters?.includes("temperature"))
+        result.temperature = settings.temperature;
+    if (settings.reasoningEffort !== undefined &&
+        model.reasoning &&
+        model.compat?.supportsReasoningEffort)
+        result.reasoning_effort = settings.reasoningEffort;
+    if (settings.maxTokens !== undefined) {
+        delete result.max_completion_tokens;
+        result.max_tokens = Math.min(settings.maxTokens, model.contextWindow);
+    }
+    return result;
+}
+export function requestSettings(payload) {
+    if (!isRecord(payload))
+        return {};
+    return Object.fromEntries(["temperature", "reasoning_effort", "max_tokens", "max_completion_tokens", "top_p"]
+        .filter((key) => payload[key] !== undefined)
+        .map((key) => [key, payload[key]]));
+}

package/dist/models.js

@@ -0,0 +1,62 @@
+export const BASE_URL = "https://api.tokenfactory.nebius.com/v1";
+export function isRecord(value) {
+    return typeof value === "object" && value !== null && !Array.isArray(value);
+}
+/** Only retain fields we understand: no credentials or arbitrary server data reach the cache. */
+export function parseModels(payload) {
+    if (!isRecord(payload) || payload.object !== "list" || !Array.isArray(payload.data)) {
+        throw new Error("Malformed Token Factory model list: expected { object: 'list', data: [...] }");
+    }
+    const models = new Map();
+    for (const entry of payload.data) {
+        if (!isRecord(entry) ||
+            typeof entry.id !== "string" ||
+            !entry.id.trim() ||
+            entry.id.length > 512 ||
+            [...entry.id].some((char) => /\s/.test(char) || char.charCodeAt(0) < 32 || char.charCodeAt(0) === 127)) {
+            throw new Error("Malformed Token Factory model list: invalid model identifier");
+        }
+        if (entry.status != null && entry.status !== "active")
+            continue;
+        const modality = isRecord(entry.architecture) ? entry.architecture.modality : undefined;
+        // Nebius also serves embeddings and image generation. Only expose text-output models.
+        if (typeof modality === "string" && !/->text$/.test(modality))
+            continue;
+        const parameters = Array.isArray(entry.supported_sampling_parameters)
+            ? entry.supported_sampling_parameters
+            : [];
+        const features = Array.isArray(entry.supported_features) ? entry.supported_features : [];
+        const effort = parameters.includes("reasoning_effort");
+        const contextWindow = positiveInteger(entry.context_length) ?? 32768;
+        models.set(entry.id, {
+            id: entry.id,
+            name: typeof entry.name === "string" && entry.name.trim() ? entry.name : entry.id,
+            provider: "nebius",
+            api: "openai-completions",
+            baseUrl: BASE_URL,
+            reasoning: effort || features.includes("reasoning"),
+            input: typeof modality === "string" && modality.split("->")[0]?.includes("image")
+                ? ["text", "image"]
+                : ["text"],
+            nebiusSupportedParameters: parameters.filter((value) => typeof value === "string"),
+            contextWindow,
+            // No documented output-token limit.
+            maxTokens: Math.min(4096, Math.max(1, Math.floor(contextWindow / 4))),
+            // Required by Pi's Model type; unused by our benchmark.
+            cost: { input: 0, output: 0, cacheRead: 0, cacheWrite: 0 },
+            compat: {
+                supportsStore: false,
+                supportsDeveloperRole: false,
+                supportsStrictMode: false,
+                supportsOpenAIGrammarTools: false,
+                supportsReasoningEffort: effort,
+                supportsUsageInStreaming: true,
+                maxTokensField: "max_tokens",
+            },
+        });
+    }
+    return [...models.values()].sort((a, b) => a.id.localeCompare(b.id));
+}
+function positiveInteger(value) {
+    return typeof value === "number" && Number.isSafeInteger(value) && value > 0 ? value : undefined;
+}

package/dist/provider.js

@@ -0,0 +1,48 @@
+import { createProvider } from "@earendil-works/pi-ai";
+import { openAICompletionsApi } from "@earendil-works/pi-ai/compat";
+import { withErrorDetails } from "./errors.js";
+import { applyRequestSettings } from "./model-settings.js";
+import { BASE_URL } from "./models.js";
+export function nebiusProvider(models, settings = {}, onRequest) {
+    const adapter = openAICompletionsApi();
+    const api = {
+        stream: (model, context, options) => adapter.stream(model, context, {
+            ...options,
+            onPayload: async (payload, selected) => {
+                const previous = await options?.onPayload?.(payload, selected);
+                const adjusted = applyRequestSettings(previous ?? payload, model, settings[model.id]);
+                onRequest?.(adjusted);
+                return adjusted;
+            },
+            fetch: withErrorDetails(options?.fetch ?? globalThis.fetch),
+        }),
+        streamSimple: (model, context, options) => adapter.streamSimple(model, context, {
+            ...options,
+            onPayload: async (payload, selected) => {
+                const previous = await options?.onPayload?.(payload, selected);
+                const adjusted = applyRequestSettings(previous ?? payload, model, settings[model.id]);
+                onRequest?.(adjusted);
+                return adjusted;
+            },
+            fetch: withErrorDetails(options?.fetch ?? globalThis.fetch),
+        }),
+    };
+    return createProvider({
+        id: "nebius",
+        name: "Nebius Token Factory",
+        baseUrl: BASE_URL,
+        models,
+        api,
+        auth: {
+            apiKey: {
+                name: "NEBIUS_API_KEY",
+                // Ambient-only auth: no login flow and no credential persistence.
+                async resolve({ ctx, signal }) {
+                    signal.throwIfAborted();
+                    const key = (await ctx.env("NEBIUS_API_KEY"))?.trim();
+                    return key ? { auth: { apiKey: key }, source: "NEBIUS_API_KEY" } : undefined;
+                },
+            },
+        },
+    });
+}

package/docs/benchmark-research.md

@@ -0,0 +1,35 @@
+# Benchmark research notes
+
+Inspected 2026-09-16. Pi published packages: **0.85.1**. Current repository head during inspection: `60e7e76bd7ea25cad1dd6f3f1ce0d18814a42759`. Nebius OpenAPI version: **20260910-cd76b4886**.
+
+## Pi contracts inspected
+
+- [SDK guide](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/docs/sdk.md) and [SDK implementation](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/sdk.ts): session creation, isolated resource loaders, normal built-in tools, configured runtime, and in-memory sessions.
+- [Extension event types](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/extensions/types.ts): `before_agent_start`, `before_provider_request`, `after_provider_response`, `turn_start`, `message_end`, and tool execution events. Request hooks permit transformation, but this benchmark does not return any modifications.
+- [AgentSession](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/agent-session.ts): `subscribe`, `prompt`, `abort`, retry events, compaction events, `agent_settled`, and effective thinking level.
+- [System prompt builder](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/system-prompt.ts): includes cwd even when a custom system prompt is supplied. The SDK resolves relative cwd before constructing the session. This drove the sequential fixed-active-path design.
+- [Model runtime](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/model-runtime.ts) and [provider contracts](https://github.com/earendil-works/pi/blob/main/packages/ai/src/models.ts): native provider registration and stream options with injected `fetch`.
+- [Chat Completions adapter](https://github.com/earendil-works/pi/blob/main/packages/ai/src/api/openai-completions.ts): parses streamed usage, subtracts cache reads/writes from Pi's `usage.input`, tracks reasoning as an output subset, and can default missing usage/detail fields to zero. Raw provider usage is observed to preserve the distinction between absent and zero.
+- [Telemetry package](https://github.com/earendil-works/pi/tree/main/packages/telemetry): span context/start/end/event contracts exist. Client spans do not establish server-only generation timing or guarantee a usage measurement for every HTTP retry. The benchmark uses public events and provider transport injection instead of installing a telemetry exporter or patching Pi core.
+- [Bash tool](https://github.com/earendil-works/pi/blob/main/packages/coding-agent/src/core/tools/bash.ts): cancellation kills process trees, but shell processes can be detached. The runner asks Pi to abort first, then applies a bounded process-tree cleanup fallback. POSIX-only behavior is documented.
+
+`before_provider_request` counts logical adapter calls, not necessarily every HTTP attempt. `after_provider_response` occurs after successful response acquisition and lacks raw stream usage. Neither alone satisfies request-level accounting across retries. The observer therefore instruments the existing adapter's fetch call and passes request arguments and body bytes through unchanged. Session events provide separate turn/tool boundaries; requests during default Pi compaction are labeled and included.
+
+## Nebius usage and models
+
+- [Chat Completions API](https://docs.tokenfactory.nebius.com/api-reference/inference/create-chat-completion) and [OpenAPI schema](https://api.tokenfactory.nebius.com/openapi.json): `Usage` requires `prompt_tokens`, `completion_tokens`, and `total_tokens`; optional prompt details include `cached_tokens`, and optional completion details include `reasoning_tokens`. Streaming supports `stream_options.include_usage`, which the existing provider already enables.
+- [List models](https://docs.tokenfactory.nebius.com/api-reference/models/list-models): `verbose=true` returns rich metadata, including context length, modality and optional capabilities. `created` and response fingerprints do not establish an immutable model revision. No revision field is documented in `RichModel`.
+
+No live key was available. Documentation defines the supported fields; tests exercise them using schema-shaped fixtures. No fixture is presented as a recording of an authenticated Nebius account response.
+
+## Agentic benchmark methodology
+
+Read the official [cookbook article](https://dev.nebius.com/cookbook/agent-cost-benchmark) and its [implementation](https://github.com/nebius/token-factory-cookbook/blob/main/agents/agent-cost-comparison-1/agent_cost_comparison_1.py).
+
+Its methodology runs the same data-analysis task through a filesystem agent with per-model input copies, validates generated JSON against expected values, accumulates message usage/tool counts,. It retains recoverable usage after errors/timeouts rather than discarding failed runs. The Pi runner retains these methodological choices while using coding fixtures, external executable validators, and physical-request traces.
+
+The cookbook uses Deep Agents and includes a model-specific harness-middleware workaround. This project copies neither the agent framework nor that behavioral change. Our repetitions, timing boundaries, and unknown-usage handling are explicit so comparisons can be audited.
+
+## Measurement limits
+
+Provider-reported tokens are exact as reported, not independently audited billing data. Unknown fields remain unknown. Client timings cannot isolate server compute or pure agent overhead. The last submitted prompt is measurable when usage is returned; final conversation token size is not. Model alias drift, serving hardware/load, account quotas, caches, and upstream updates remain external reproducibility limits.