pi-nebius 0.3.0

package/dist/benchmark/instrumentation.js

@@ -0,0 +1,296 @@
+import { createHash } from "node:crypto";
+import { isRecord } from "../models.js";
+export const hash = (text) => createHash("sha256").update(text).digest("hex");
+const count = (value) => typeof value === "number" && Number.isSafeInteger(value) && value >= 0 ? value : null;
+export function reportedUsage(value) {
+    if (!isRecord(value))
+        return null;
+    const prompt = isRecord(value.prompt_tokens_details) ? value.prompt_tokens_details : {};
+    const completion = isRecord(value.completion_tokens_details)
+        ? value.completion_tokens_details
+        : {};
+    return {
+        inputTokens: count(value.prompt_tokens),
+        outputTokens: count(value.completion_tokens),
+        cachedInputTokens: count(prompt.cached_tokens ?? value.prompt_cache_hit_tokens ?? value.cached_tokens),
+        reasoningTokens: count(completion.reasoning_tokens),
+        totalTokens: count(value.total_tokens),
+    };
+}
+export function emptyObservation() {
+    return {
+        requests: [],
+        tools: [],
+        agentTurns: 0,
+        assistantMessages: 0,
+        toolCalls: 0,
+        toolCallsByType: {},
+        compactions: 0,
+        retries: 0,
+        agentStartedAtMs: null,
+        agentEndedAtMs: null,
+        systemPromptHash: null,
+        lastAssistantStopReason: null,
+        errors: [],
+    };
+}
+export function redactor(secrets) {
+    return (value) => {
+        let safe = value;
+        for (const secret of secrets.filter(Boolean))
+            safe = safe.split(secret).join("[REDACTED]");
+        return safe.replace(/Bearer\s+[^\s"']+/gi, "Bearer [REDACTED]");
+    };
+}
+/** Observer only: never returns modified payloads, tools, messages, or stream bytes. */
+export class Instrumentation {
+    state = emptyObservation();
+    compacting = false;
+    update;
+    now;
+    redact;
+    constructor(update = () => { }, now = () => performance.now(), redact = (text) => text) {
+        this.update = update;
+        this.now = now;
+        this.redact = redact;
+    }
+    systemPrompt(prompt) {
+        this.state.systemPromptHash = hash(prompt);
+        this.publish();
+    }
+    publish() {
+        this.update(this.state);
+    }
+    onEvent(event) {
+        const at = this.now();
+        switch (event.type) {
+            case "agent_start":
+                this.state.agentStartedAtMs ??= at;
+                break;
+            case "agent_settled":
+                this.state.agentEndedAtMs = at;
+                break;
+            case "turn_start":
+                this.state.agentTurns++;
+                break;
+            case "compaction_start":
+                this.compacting = true;
+                this.state.compactions++;
+                break;
+            case "compaction_end":
+                this.compacting = false;
+                break;
+            case "auto_retry_start":
+                this.state.retries++;
+                break;
+            case "tool_execution_start":
+                this.state.tools.push({
+                    id: event.toolCallId,
+                    name: event.toolName,
+                    startedAtMs: at,
+                    endedAtMs: null,
+                    isError: null,
+                    error: null,
+                });
+                break;
+            case "tool_execution_end": {
+                const tool = this.state.tools.find((item) => item.id === event.toolCallId);
+                if (tool) {
+                    tool.endedAtMs = at;
+                    tool.isError = event.isError;
+                }
+                break;
+            }
+            case "message_end": {
+                const message = event.message;
+                if (message.role === "assistant") {
+                    this.state.assistantMessages++;
+                    this.state.lastAssistantStopReason = message.stopReason;
+                    if (message.errorMessage)
+                        this.state.errors.push(this.redact(message.errorMessage).slice(0, 8000));
+                    for (const block of message.content)
+                        if (block.type === "toolCall") {
+                            this.state.toolCalls++;
+                            this.state.toolCallsByType[block.name] =
+                                (this.state.toolCallsByType[block.name] ?? 0) + 1;
+                        }
+                }
+                else if (message.role === "toolResult") {
+                    let tool = this.state.tools.find((item) => item.id === message.toolCallId);
+                    if (!tool) {
+                        tool = {
+                            id: message.toolCallId,
+                            name: message.toolName,
+                            startedAtMs: null,
+                            endedAtMs: at,
+                            isError: message.isError,
+                            error: null,
+                        };
+                        this.state.tools.push(tool);
+                    }
+                    tool.isError = message.isError;
+                    if (message.isError)
+                        tool.error = this.redact(message.content
+                            .filter((item) => item.type === "text")
+                            .map((item) => item.text)
+                            .join("\n")).slice(0, 4000);
+                }
+                break;
+            }
+            default:
+                return;
+        }
+        this.publish();
+    }
+    observeFetch(fetcher) {
+        return async (input, init) => {
+            const url = new URL(input instanceof Request ? input.url : input.toString());
+            if (!url.pathname.endsWith("/chat/completions"))
+                return fetcher(input, init);
+            const trace = {
+                request: this.state.requests.length + 1,
+                purpose: this.compacting ? "compaction" : "agent",
+                startedAtMs: this.now(),
+                endedAtMs: null,
+                firstContentAtMs: null,
+                status: null,
+                requestId: null,
+                servedModel: null,
+                systemFingerprint: null,
+                finishReason: null,
+                usage: null,
+                error: null,
+                streamComplete: false,
+            };
+            this.state.requests.push(trace);
+            this.publish();
+            try {
+                const response = await fetcher(input, init);
+                trace.status = response.status;
+                trace.requestId = response.headers.get("x-request-id");
+                this.publish();
+                if (!response.body) {
+                    trace.endedAtMs = this.now();
+                    this.publish();
+                    return response;
+                }
+                const decoder = new TextDecoder();
+                let buffer = "";
+                let dropped = false;
+                const consume = (data) => {
+                    if (data.trim() === "[DONE]") {
+                        trace.streamComplete = true;
+                        return;
+                    }
+                    let chunk;
+                    try {
+                        chunk = JSON.parse(data);
+                    }
+                    catch {
+                        return;
+                    }
+                    if (!isRecord(chunk))
+                        return;
+                    if (typeof chunk.model === "string")
+                        trace.servedModel = chunk.model;
+                    if (typeof chunk.system_fingerprint === "string")
+                        trace.systemFingerprint = chunk.system_fingerprint;
+                    if (chunk.usage)
+                        trace.usage = reportedUsage(chunk.usage);
+                    for (const choice of Array.isArray(chunk.choices) ? chunk.choices : []) {
+                        if (!isRecord(choice))
+                            continue;
+                        if (typeof choice.finish_reason === "string")
+                            trace.finishReason = choice.finish_reason;
+                        if (choice.usage)
+                            trace.usage = reportedUsage(choice.usage);
+                        const delta = isRecord(choice.delta) ? choice.delta : {};
+                        if (trace.firstContentAtMs === null &&
+                            ([delta.content, delta.reasoning_content, delta.reasoning].some((value) => typeof value === "string" && value.length > 0) ||
+                                (Array.isArray(delta.tool_calls) &&
+                                    delta.tool_calls.some((call) => {
+                                        if (!isRecord(call) || !isRecord(call.function))
+                                            return false;
+                                        return [call.function.name, call.function.arguments].some((value) => typeof value === "string" && value.length > 0);
+                                    }))))
+                            trace.firstContentAtMs = this.now();
+                    }
+                };
+                const ingest = (bytes) => {
+                    if (dropped)
+                        return;
+                    const previous = JSON.stringify([trace.usage, trace.firstContentAtMs]);
+                    buffer += decoder.decode(bytes, { stream: true });
+                    if (buffer.length > 1024 * 1024) {
+                        dropped = true;
+                        buffer = "";
+                        trace.error = "Instrumentation frame exceeded 1 MiB; usage may be incomplete";
+                        return;
+                    }
+                    // Line parser handles arbitrary TCP chunking, LF and CRLF. JSON payloads
+                    // are contained in SSE data lines for Nebius's Chat Completions protocol.
+                    let newline = buffer.indexOf("\n");
+                    while (newline >= 0) {
+                        const line = buffer.slice(0, newline).replace(/\r$/, "");
+                        buffer = buffer.slice(newline + 1);
+                        if (line.startsWith("data:"))
+                            consume(line.slice(5).trimStart());
+                        newline = buffer.indexOf("\n");
+                    }
+                    if (JSON.stringify([trace.usage, trace.firstContentAtMs]) !== previous)
+                        this.publish();
+                };
+                const reader = response.body.getReader();
+                const finish = () => {
+                    trace.endedAtMs = this.now();
+                    this.publish();
+                };
+                const body = new ReadableStream({
+                    pull: async (controller) => {
+                        try {
+                            const result = await reader.read();
+                            if (result.done) {
+                                buffer += decoder.decode();
+                                if (buffer.startsWith("data:"))
+                                    consume(buffer.slice(5).trim());
+                                if (!response.ok)
+                                    trace.error = this.redact(`HTTP ${response.status}: ${buffer}`).slice(0, 8000);
+                                finish();
+                                controller.close();
+                                return;
+                            }
+                            // Forward exactly the original bytes. Observation is best effort and cannot fail inference.
+                            try {
+                                ingest(result.value);
+                            }
+                            catch {
+                                trace.error = "Instrumentation could not decode response metadata";
+                            }
+                            controller.enqueue(result.value);
+                        }
+                        catch (error) {
+                            trace.error = this.redact(String(error)).slice(0, 8000);
+                            finish();
+                            controller.error(error);
+                        }
+                    },
+                    cancel: async (reason) => {
+                        finish();
+                        await reader.cancel(reason);
+                    },
+                });
+                return new Response(body, {
+                    status: response.status,
+                    statusText: response.statusText,
+                    headers: response.headers,
+                });
+            }
+            catch (error) {
+                trace.error = this.redact(String(error)).slice(0, 8000);
+                trace.endedAtMs = this.now();
+                this.publish();
+                throw error;
+            }
+        };
+    }
+}

package/dist/benchmark/metrics.js

@@ -0,0 +1,78 @@
+export function tokenTotals(requests) {
+    const sum = (field) => requests.reduce((total, request) => total + (request.usage?.[field] ?? 0), 0);
+    const complete = (field) => requests.length > 0 && requests.every((request) => request.usage?.[field] != null);
+    const input = complete("inputTokens") ? sum("inputTokens") : null;
+    const last = requests.filter((request) => request.purpose === "agent").at(-1)?.usage?.inputTokens ?? null;
+    return {
+        cumulativeInputTokens: input,
+        cumulativeOutputTokens: complete("outputTokens") ? sum("outputTokens") : null,
+        cachedInputTokens: complete("cachedInputTokens") ? sum("cachedInputTokens") : null,
+        reasoningTokens: complete("reasoningTokens") ? sum("reasoningTokens") : null,
+        observedInputTokens: sum("inputTokens"),
+        observedOutputTokens: sum("outputTokens"),
+        requestsWithUsage: requests.filter((request) => request.usage?.inputTokens != null && request.usage.outputTokens != null).length,
+        usageComplete: complete("inputTokens") && complete("outputTokens"),
+        lastRequestInputTokens: last,
+        finalContextSizeTokens: null,
+        inputAmplification: null,
+        inputAmplificationVsLastRequest: input !== null && last !== null && last > 0 ? input / last : null,
+    };
+}
+export function distribution(values) {
+    const known = values
+        .filter((value) => value !== null && Number.isFinite(value))
+        .sort((a, b) => a - b);
+    const count = known.length;
+    if (!count)
+        return {
+            count: 0,
+            missing: values.length,
+            mean: null,
+            median: null,
+            min: null,
+            max: null,
+            standardDeviation: null,
+        };
+    const mean = known.reduce((a, b) => a + b, 0) / count;
+    return {
+        count,
+        missing: values.length - count,
+        mean,
+        median: ((known[Math.floor((count - 1) / 2)] ?? 0) + (known[Math.floor(count / 2)] ?? 0)) / 2,
+        min: known[0] ?? null,
+        max: known.at(-1) ?? null,
+        standardDeviation: Math.sqrt(known.reduce((total, value) => total + (value - mean) ** 2, 0) / count),
+    };
+}
+export function aggregate(runs) {
+    return [...new Set(runs.map((run) => run.model))].map((model) => {
+        const group = runs.filter((run) => run.model === model);
+        const successes = group.filter((run) => run.success).length;
+        return {
+            model,
+            runs: group.length,
+            successes,
+            successRate: group.every((run) => run.validation?.checked === false)
+                ? null
+                : successes / group.length,
+            wallTimeMs: distribution(group.map((run) => run.wallTimeMs)),
+            inputTokens: distribution(group.map((run) => run.tokens.cumulativeInputTokens)),
+            outputTokens: distribution(group.map((run) => run.tokens.cumulativeOutputTokens)),
+            turns: distribution(group.map((run) => run.agentTurns)),
+            toolCalls: distribution(group.map((run) => run.toolCalls)),
+        };
+    });
+}
+/** Union, rather than sum, avoids double-counting parallel tool/request intervals. */
+export function intervalDuration(intervals) {
+    const ordered = intervals
+        .filter((item) => item.startedAtMs !== null && item.endedAtMs !== null)
+        .sort((a, b) => a.startedAtMs - b.startedAtMs);
+    let total = 0;
+    let end = 0;
+    for (const item of ordered) {
+        total += Math.max(0, item.endedAtMs - Math.max(end, item.startedAtMs));
+        end = Math.max(end, item.endedAtMs);
+    }
+    return total;
+}

package/dist/benchmark/process.js

@@ -0,0 +1,122 @@
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
+export function cleanEnvironment() {
+    return { PATH: process.env.PATH, LANG: "C.UTF-8", LC_ALL: "C.UTF-8", TZ: "UTC" };
+}
+export function terminateGroup(pid, signal = "SIGTERM") {
+    try {
+        process.kill(-pid, signal);
+    }
+    catch {
+        try {
+            process.kill(pid, signal);
+        }
+        catch {
+            /* Already exited. */
+        }
+    }
+}
+export async function descendants(pid) {
+    try {
+        const { stdout } = await promisify(execFile)("ps", ["-axo", "pid=,ppid="], {
+            maxBuffer: 4 * 1024 * 1024,
+        });
+        const rows = stdout
+            .trim()
+            .split("\n")
+            .map((line) => line.trim().split(/\s+/).map(Number));
+        const found = new Set([pid]);
+        for (let changed = true; changed;) {
+            changed = false;
+            for (const [child, parent] of rows)
+                if (child && parent && found.has(parent) && !found.has(child)) {
+                    found.add(child);
+                    changed = true;
+                }
+        }
+        found.delete(pid);
+        return [...found].reverse();
+    }
+    catch {
+        return [];
+    }
+}
+export async function runCommand(command, cwd, timeoutMs, signal, redact = (text) => text) {
+    const started = performance.now();
+    if (signal?.aborted)
+        return {
+            command,
+            exitCode: null,
+            signal: "SIGINT",
+            durationMs: 0,
+            timedOut: false,
+            output: "Cancelled before validation",
+            outputTruncated: false,
+        };
+    return new Promise((resolve) => {
+        const child = spawn(command.command, command.args, {
+            cwd,
+            env: cleanEnvironment(),
+            detached: true,
+            stdio: ["ignore", "pipe", "pipe"],
+        });
+        const chunks = [];
+        let bytes = 0;
+        let truncated = false;
+        let timedOut = false;
+        let settled = false;
+        let force;
+        const stop = () => {
+            if (!child.pid)
+                return;
+            terminateGroup(child.pid);
+            force ??= setTimeout(() => {
+                if (child.pid)
+                    terminateGroup(child.pid, "SIGKILL");
+            }, 1000);
+        };
+        const timer = setTimeout(() => {
+            timedOut = true;
+            stop();
+        }, timeoutMs);
+        signal?.addEventListener("abort", stop, { once: true });
+        const capture = (data) => {
+            const remaining = 256 * 1024 - bytes;
+            if (data.length > remaining)
+                truncated = true;
+            if (remaining > 0) {
+                chunks.push(data.subarray(0, remaining));
+                bytes += Math.min(remaining, data.length);
+            }
+        };
+        child.stdout.on("data", capture);
+        child.stderr.on("data", capture);
+        const finish = (code, exitSignal, error) => {
+            if (settled)
+                return;
+            settled = true;
+            clearTimeout(timer);
+            if (force)
+                clearTimeout(force);
+            signal?.removeEventListener("abort", stop);
+            // Remove descendants that kept stdio open after their command exited.
+            if (child.pid)
+                terminateGroup(child.pid, "SIGKILL");
+            resolve({
+                command,
+                exitCode: code,
+                signal: exitSignal,
+                durationMs: performance.now() - started,
+                timedOut,
+                output: redact(error ?? Buffer.concat(chunks).toString("utf8")),
+                outputTruncated: truncated,
+            });
+        };
+        child.on("error", (error) => finish(null, null, String(error)));
+        child.on("exit", (code, exitSignal) => {
+            if (child.pid)
+                terminateGroup(child.pid, "SIGKILL");
+            child.once("close", () => finish(code, exitSignal));
+        });
+    });
+}

package/dist/benchmark/project.js

@@ -0,0 +1,70 @@
+import { execFileSync } from "node:child_process";
+import { cp, lstat, mkdir, readdir } from "node:fs/promises";
+import { dirname, join } from "node:path";
+const excluded = (path) => path
+    .split(/[\\/]/)
+    .some((name) => [
+    ".git",
+    ".pi",
+    "node_modules",
+    "benchmark-results",
+    "dist",
+    "build",
+    "coverage",
+    ".next",
+].includes(name) ||
+    name === ".env" ||
+    name.startsWith(".env.") ||
+    /\.(pem|key)$/.test(name));
+/** Snapshot working files, respecting Git ignores, without dependency trees or known credential files. */
+export async function snapshotProject(source, target) {
+    let files;
+    try {
+        files = execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], {
+            cwd: source,
+            encoding: "utf8",
+            stdio: ["ignore", "pipe", "ignore"],
+            maxBuffer: 8 * 1024 * 1024,
+        })
+            .split("\0")
+            .filter(Boolean);
+    }
+    catch {
+        const walk = async (prefix = "") => {
+            const found = [];
+            for (const entry of await readdir(join(source, prefix), { withFileTypes: true })) {
+                const path = join(prefix, entry.name);
+                if (excluded(path))
+                    continue;
+                if (entry.isDirectory())
+                    found.push(...(await walk(path)));
+                else
+                    found.push(path);
+            }
+            return found;
+        };
+        files = await walk();
+    }
+    await mkdir(target, { recursive: true, mode: 0o700 });
+    let bytes = 0;
+    let copied = 0;
+    for (const path of new Set(files)) {
+        if (excluded(path))
+            continue;
+        const info = await lstat(join(source, path)).catch((error) => {
+            if (error.code === "ENOENT")
+                return undefined;
+            throw error;
+        });
+        if (!info)
+            continue;
+        if (!info.isFile())
+            throw new Error(`Cannot snapshot ${path}: only regular project files are supported.`);
+        bytes += info.size;
+        if (++copied > 10000 || bytes > 50 * 1024 * 1024)
+            throw new Error("Project snapshot exceeds 10,000 files or 50 MiB. Run Pi from a smaller project directory.");
+        await mkdir(dirname(join(target, path)), { recursive: true, mode: 0o700 });
+        await cp(join(source, path), join(target, path));
+    }
+    return copied;
+}

package/dist/benchmark/report.js

@@ -0,0 +1,94 @@
+import { distribution } from "./metrics.js";
+/** Client-observed first generated content on the first agent request, not worker startup. */
+export function observedTtftMs(run) {
+    const first = run.observation.requests.find((request) => request.purpose === "agent");
+    if (first?.firstContentAtMs == null)
+        return null;
+    const elapsed = first.firstContentAtMs - first.startedAtMs;
+    return elapsed >= 0 ? elapsed : null;
+}
+/** End-to-end output rate, including tools, network, setup, and validation. */
+export function outputThroughput(runs) {
+    if (!runs.length ||
+        runs.some((run) => run.tokens.cumulativeOutputTokens === null || run.wallTimeMs <= 0))
+        return null;
+    const tokens = runs.reduce((sum, run) => sum + (run.tokens.cumulativeOutputTokens ?? 0), 0);
+    return tokens / (runs.reduce((sum, run) => sum + run.wallTimeMs, 0) / 1000);
+}
+function failureDetails(run) {
+    const failed = run.validation.commands.find((command) => command.exitCode !== 0 || command.timedOut);
+    const output = failed?.output ?? "";
+    const reason = output.match(/\berror: \|-\n\s+([^\n]+)/)?.[1] ?? output.match(/\berror: (.+)/)?.[1];
+    const named = output.match(/not ok \d+ - (.+)/)?.[1];
+    const location = output.match(/TestContext[^\n]*\/([^/\n]+\.(?:mjs|js|ts):\d+:\d+)/)?.[1];
+    const detail = reason && reason !== "|-" ? reason : named;
+    return [run.errors[0] || detail, location]
+        .filter(Boolean)
+        .join("; ")
+        .replace(/\s+/g, " ")
+        .slice(0, 350);
+}
+export function terminalReport(results) {
+    const unchecked = results.definition.validationMode === "none";
+    const groups = results.aggregates;
+    const runs = groups.map((group) => results.runs.filter((run) => run.model === group.model));
+    const number = (value, decimals = 1) => value === null
+        ? "n/a"
+        : value.toLocaleString("en-US", {
+            minimumFractionDigits: decimals,
+            maximumFractionDigits: decimals,
+        });
+    const seconds = (value) => value === null ? "n/a" : `${number(value / 1000, 2)} s`;
+    const rows = [
+        ["Metric", ...groups.map((group) => group.model)],
+        [
+            unchecked ? "Finished runs (not validated)" : "Validated success",
+            ...groups.map((group, index) => `${unchecked ? runs[index]?.filter((run) => run.failure === null).length : group.successes}/${group.runs}`),
+        ],
+        ["Input tokens / run (mean)", ...groups.map((group) => number(group.inputTokens.mean, 0))],
+        ["Output tokens / run (mean)", ...groups.map((group) => number(group.outputTokens.mean, 0))],
+        ["Task duration (median)", ...groups.map((group) => seconds(group.wallTimeMs.median))],
+        [
+            "Observed TTFT (median)",
+            ...runs.map((group) => seconds(distribution(group.map(observedTtftMs)).median)),
+        ],
+        ["Throughput (output tokens/s)", ...runs.map((group) => number(outputThroughput(group)))],
+        ["Agent turns / run (mean)", ...groups.map((group) => number(group.turns.mean))],
+        ["Tool calls / run (mean)", ...groups.map((group) => number(group.toolCalls.mean))],
+    ];
+    const widths = rows[0]?.map((_, index) => Math.max(...rows.map((row) => row[index]?.length ?? 0))) ?? [];
+    const separator = widths.map((width) => "─".repeat(width)).join("─┼─");
+    const table = rows.map((row) => row
+        .map((cell, index) => cell.padEnd(widths[index] ?? 0))
+        .join(" │ ")
+        .trimEnd());
+    table.splice(1, 0, separator);
+    const failures = results.runs.filter((run) => run.failure !== null);
+    return [
+        `Benchmark: ${results.definition.name}`,
+        `Completed: ${results.runs.length}/${results.plannedRuns} (${results.status})`,
+        ...(unchecked
+            ? ["Correctness: not checked. Finished means execution completed without an error."]
+            : []),
+        "",
+        ...table,
+        "",
+        "Tokens include every request in a run. Failed runs contribute to these metrics too.",
+        "TTFT: first agent request start → first streamed text/reasoning/tool content; excludes worker startup.",
+        "Stream chunks may contain multiple tokens; TTFT is client-observed, not server token timing.",
+        "Throughput: total output tokens ÷ total task seconds, including network, tools, and validation.",
+        "Unknown values are n/a. Means/medians exclude missing values; throughput needs complete output usage for every run.",
+        ...(failures.length
+            ? [
+                "",
+                "Failed runs:",
+                ...failures.map((run) => {
+                    const details = failureDetails(run);
+                    return `- ${run.model} #${run.run}: ${run.failure}${details ? ` — ${details}` : ""}`;
+                }),
+            ]
+            : []),
+        "",
+        "Detailed per-run measurements and validator output: results.json.",
+    ].join("\n");
+}