pi-nebius 0.3.0

package/src/benchmark/metrics.ts

@@ -0,0 +1,101 @@
+import type {
+  Distribution,
+  ModelAggregate,
+  RequestTrace,
+  RunResult,
+  TokenTotals,
+} from "./types.ts";
+
+export function tokenTotals(requests: RequestTrace[]): TokenTotals {
+  const sum = (field: "inputTokens" | "outputTokens" | "cachedInputTokens" | "reasoningTokens") =>
+    requests.reduce((total, request) => total + (request.usage?.[field] ?? 0), 0);
+  const complete = (
+    field: "inputTokens" | "outputTokens" | "cachedInputTokens" | "reasoningTokens",
+  ) => requests.length > 0 && requests.every((request) => request.usage?.[field] != null);
+  const input = complete("inputTokens") ? sum("inputTokens") : null;
+  const last =
+    requests.filter((request) => request.purpose === "agent").at(-1)?.usage?.inputTokens ?? null;
+  return {
+    cumulativeInputTokens: input,
+    cumulativeOutputTokens: complete("outputTokens") ? sum("outputTokens") : null,
+    cachedInputTokens: complete("cachedInputTokens") ? sum("cachedInputTokens") : null,
+    reasoningTokens: complete("reasoningTokens") ? sum("reasoningTokens") : null,
+    observedInputTokens: sum("inputTokens"),
+    observedOutputTokens: sum("outputTokens"),
+    requestsWithUsage: requests.filter(
+      (request) => request.usage?.inputTokens != null && request.usage.outputTokens != null,
+    ).length,
+    usageComplete: complete("inputTokens") && complete("outputTokens"),
+    lastRequestInputTokens: last,
+    finalContextSizeTokens: null,
+    inputAmplification: null,
+    inputAmplificationVsLastRequest:
+      input !== null && last !== null && last > 0 ? input / last : null,
+  };
+}
+
+export function distribution(values: Array<number | null>): Distribution {
+  const known = values
+    .filter((value): value is number => value !== null && Number.isFinite(value))
+    .sort((a, b) => a - b);
+  const count = known.length;
+  if (!count)
+    return {
+      count: 0,
+      missing: values.length,
+      mean: null,
+      median: null,
+      min: null,
+      max: null,
+      standardDeviation: null,
+    };
+  const mean = known.reduce((a, b) => a + b, 0) / count;
+  return {
+    count,
+    missing: values.length - count,
+    mean,
+    median: ((known[Math.floor((count - 1) / 2)] ?? 0) + (known[Math.floor(count / 2)] ?? 0)) / 2,
+    min: known[0] ?? null,
+    max: known.at(-1) ?? null,
+    standardDeviation: Math.sqrt(
+      known.reduce((total, value) => total + (value - mean) ** 2, 0) / count,
+    ),
+  };
+}
+export function aggregate(runs: RunResult[]): ModelAggregate[] {
+  return [...new Set(runs.map((run) => run.model))].map((model) => {
+    const group = runs.filter((run) => run.model === model);
+    const successes = group.filter((run) => run.success).length;
+    return {
+      model,
+      runs: group.length,
+      successes,
+      successRate: group.every((run) => run.validation?.checked === false)
+        ? null
+        : successes / group.length,
+      wallTimeMs: distribution(group.map((run) => run.wallTimeMs)),
+      inputTokens: distribution(group.map((run) => run.tokens.cumulativeInputTokens)),
+      outputTokens: distribution(group.map((run) => run.tokens.cumulativeOutputTokens)),
+      turns: distribution(group.map((run) => run.agentTurns)),
+      toolCalls: distribution(group.map((run) => run.toolCalls)),
+    };
+  });
+}
+/** Union, rather than sum, avoids double-counting parallel tool/request intervals. */
+export function intervalDuration(
+  intervals: Array<{ startedAtMs: number | null; endedAtMs: number | null }>,
+): number {
+  const ordered = intervals
+    .filter(
+      (item): item is { startedAtMs: number; endedAtMs: number } =>
+        item.startedAtMs !== null && item.endedAtMs !== null,
+    )
+    .sort((a, b) => a.startedAtMs - b.startedAtMs);
+  let total = 0;
+  let end = 0;
+  for (const item of ordered) {
+    total += Math.max(0, item.endedAtMs - Math.max(end, item.startedAtMs));
+    end = Math.max(end, item.endedAtMs);
+  }
+  return total;
+}

package/src/benchmark/process.ts

@@ -0,0 +1,120 @@
+import { execFile, spawn } from "node:child_process";
+import { promisify } from "node:util";
+import type { Command, CommandResult } from "./types.ts";
+
+export function cleanEnvironment(): NodeJS.ProcessEnv {
+  return { PATH: process.env.PATH, LANG: "C.UTF-8", LC_ALL: "C.UTF-8", TZ: "UTC" };
+}
+export function terminateGroup(pid: number, signal: NodeJS.Signals = "SIGTERM") {
+  try {
+    process.kill(-pid, signal);
+  } catch {
+    try {
+      process.kill(pid, signal);
+    } catch {
+      /* Already exited. */
+    }
+  }
+}
+export async function descendants(pid: number): Promise<number[]> {
+  try {
+    const { stdout } = await promisify(execFile)("ps", ["-axo", "pid=,ppid="], {
+      maxBuffer: 4 * 1024 * 1024,
+    });
+    const rows = stdout
+      .trim()
+      .split("\n")
+      .map((line) => line.trim().split(/\s+/).map(Number));
+    const found = new Set([pid]);
+    for (let changed = true; changed; ) {
+      changed = false;
+      for (const [child, parent] of rows)
+        if (child && parent && found.has(parent) && !found.has(child)) {
+          found.add(child);
+          changed = true;
+        }
+    }
+    found.delete(pid);
+    return [...found].reverse();
+  } catch {
+    return [];
+  }
+}
+export async function runCommand(
+  command: Command,
+  cwd: string,
+  timeoutMs: number,
+  signal?: AbortSignal,
+  redact: (text: string) => string = (text) => text,
+): Promise<CommandResult> {
+  const started = performance.now();
+  if (signal?.aborted)
+    return {
+      command,
+      exitCode: null,
+      signal: "SIGINT",
+      durationMs: 0,
+      timedOut: false,
+      output: "Cancelled before validation",
+      outputTruncated: false,
+    };
+  return new Promise((resolve) => {
+    const child = spawn(command.command, command.args, {
+      cwd,
+      env: cleanEnvironment(),
+      detached: true,
+      stdio: ["ignore", "pipe", "pipe"],
+    });
+    const chunks: Buffer[] = [];
+    let bytes = 0;
+    let truncated = false;
+    let timedOut = false;
+    let settled = false;
+    let force: NodeJS.Timeout | undefined;
+    const stop = () => {
+      if (!child.pid) return;
+      terminateGroup(child.pid);
+      force ??= setTimeout(() => {
+        if (child.pid) terminateGroup(child.pid, "SIGKILL");
+      }, 1000);
+    };
+    const timer = setTimeout(() => {
+      timedOut = true;
+      stop();
+    }, timeoutMs);
+    signal?.addEventListener("abort", stop, { once: true });
+    const capture = (data: Buffer) => {
+      const remaining = 256 * 1024 - bytes;
+      if (data.length > remaining) truncated = true;
+      if (remaining > 0) {
+        chunks.push(data.subarray(0, remaining));
+        bytes += Math.min(remaining, data.length);
+      }
+    };
+    child.stdout.on("data", capture);
+    child.stderr.on("data", capture);
+    const finish = (code: number | null, exitSignal: string | null, error?: string) => {
+      if (settled) return;
+      settled = true;
+      clearTimeout(timer);
+      if (force) clearTimeout(force);
+      signal?.removeEventListener("abort", stop);
+      // Remove descendants that kept stdio open after their command exited.
+      if (child.pid) terminateGroup(child.pid, "SIGKILL");
+      resolve({
+        command,
+        exitCode: code,
+        signal: exitSignal,
+        durationMs: performance.now() - started,
+        timedOut,
+        output: redact(error ?? Buffer.concat(chunks).toString("utf8")),
+        outputTruncated: truncated,
+      });
+    };
+    child.on("error", (error) => finish(null, null, String(error)));
+    child.on("exit", (code, exitSignal) => {
+      if (child.pid) terminateGroup(child.pid, "SIGKILL");
+      child.once("close", () => finish(code, exitSignal));
+    });
+  });
+}

package/src/benchmark/project.ts

@@ -0,0 +1,71 @@
+import { execFileSync } from "node:child_process";
+import { cp, lstat, mkdir, readdir } from "node:fs/promises";
+import { dirname, join } from "node:path";
+
+const excluded = (path: string) =>
+  path
+    .split(/[\\/]/)
+    .some(
+      (name) =>
+        [
+          ".git",
+          ".pi",
+          "node_modules",
+          "benchmark-results",
+          "dist",
+          "build",
+          "coverage",
+          ".next",
+        ].includes(name) ||
+        name === ".env" ||
+        name.startsWith(".env.") ||
+        /\.(pem|key)$/.test(name),
+    );
+
+/** Snapshot working files, respecting Git ignores, without dependency trees or known credential files. */
+export async function snapshotProject(source: string, target: string) {
+  let files: string[];
+  try {
+    files = execFileSync("git", ["ls-files", "--cached", "--others", "--exclude-standard", "-z"], {
+      cwd: source,
+      encoding: "utf8",
+      stdio: ["ignore", "pipe", "ignore"],
+      maxBuffer: 8 * 1024 * 1024,
+    })
+      .split("\0")
+      .filter(Boolean);
+  } catch {
+    const walk = async (prefix = ""): Promise<string[]> => {
+      const found: string[] = [];
+      for (const entry of await readdir(join(source, prefix), { withFileTypes: true })) {
+        const path = join(prefix, entry.name);
+        if (excluded(path)) continue;
+        if (entry.isDirectory()) found.push(...(await walk(path)));
+        else found.push(path);
+      }
+      return found;
+    };
+    files = await walk();
+  }
+  await mkdir(target, { recursive: true, mode: 0o700 });
+  let bytes = 0;
+  let copied = 0;
+  for (const path of new Set(files)) {
+    if (excluded(path)) continue;
+    const info = await lstat(join(source, path)).catch((error) => {
+      if (error.code === "ENOENT") return undefined;
+      throw error;
+    });
+    if (!info) continue;
+    if (!info.isFile())
+      throw new Error(`Cannot snapshot ${path}: only regular project files are supported.`);
+    bytes += info.size;
+    if (++copied > 10000 || bytes > 50 * 1024 * 1024)
+      throw new Error(
+        "Project snapshot exceeds 10,000 files or 50 MiB. Run Pi from a smaller project directory.",
+      );
+    await mkdir(dirname(join(target, path)), { recursive: true, mode: 0o700 });
+    await cp(join(source, path), join(target, path));
+  }
+  return copied;
+}

package/src/benchmark/report.ts

@@ -0,0 +1,111 @@
+import { distribution } from "./metrics.ts";
+import type { Results, RunResult } from "./types.ts";
+
+/** Client-observed first generated content on the first agent request, not worker startup. */
+export function observedTtftMs(run: RunResult): number | null {
+  const first = run.observation.requests.find((request) => request.purpose === "agent");
+  if (first?.firstContentAtMs == null) return null;
+  const elapsed = first.firstContentAtMs - first.startedAtMs;
+  return elapsed >= 0 ? elapsed : null;
+}
+
+/** End-to-end output rate, including tools, network, setup, and validation. */
+export function outputThroughput(runs: RunResult[]): number | null {
+  if (
+    !runs.length ||
+    runs.some((run) => run.tokens.cumulativeOutputTokens === null || run.wallTimeMs <= 0)
+  )
+    return null;
+  const tokens = runs.reduce((sum, run) => sum + (run.tokens.cumulativeOutputTokens ?? 0), 0);
+  return tokens / (runs.reduce((sum, run) => sum + run.wallTimeMs, 0) / 1000);
+}
+
+function failureDetails(run: RunResult): string {
+  const failed = run.validation.commands.find(
+    (command) => command.exitCode !== 0 || command.timedOut,
+  );
+  const output = failed?.output ?? "";
+  const reason =
+    output.match(/\berror: \|-\n\s+([^\n]+)/)?.[1] ?? output.match(/\berror: (.+)/)?.[1];
+  const named = output.match(/not ok \d+ - (.+)/)?.[1];
+  const location = output.match(/TestContext[^\n]*\/([^/\n]+\.(?:mjs|js|ts):\d+:\d+)/)?.[1];
+  const detail = reason && reason !== "|-" ? reason : named;
+  return [run.errors[0] || detail, location]
+    .filter(Boolean)
+    .join("; ")
+    .replace(/\s+/g, " ")
+    .slice(0, 350);
+}
+
+export function terminalReport(results: Results): string {
+  const unchecked = results.definition.validationMode === "none";
+  const groups = results.aggregates;
+  const runs = groups.map((group) => results.runs.filter((run) => run.model === group.model));
+  const number = (value: number | null, decimals = 1) =>
+    value === null
+      ? "n/a"
+      : value.toLocaleString("en-US", {
+          minimumFractionDigits: decimals,
+          maximumFractionDigits: decimals,
+        });
+  const seconds = (value: number | null) =>
+    value === null ? "n/a" : `${number(value / 1000, 2)} s`;
+  const rows = [
+    ["Metric", ...groups.map((group) => group.model)],
+    [
+      unchecked ? "Finished runs (not validated)" : "Validated success",
+      ...groups.map(
+        (group, index) =>
+          `${unchecked ? runs[index]?.filter((run) => run.failure === null).length : group.successes}/${group.runs}`,
+      ),
+    ],
+    ["Input tokens / run (mean)", ...groups.map((group) => number(group.inputTokens.mean, 0))],
+    ["Output tokens / run (mean)", ...groups.map((group) => number(group.outputTokens.mean, 0))],
+    ["Task duration (median)", ...groups.map((group) => seconds(group.wallTimeMs.median))],
+    [
+      "Observed TTFT (median)",
+      ...runs.map((group) => seconds(distribution(group.map(observedTtftMs)).median)),
+    ],
+    ["Throughput (output tokens/s)", ...runs.map((group) => number(outputThroughput(group)))],
+    ["Agent turns / run (mean)", ...groups.map((group) => number(group.turns.mean))],
+    ["Tool calls / run (mean)", ...groups.map((group) => number(group.toolCalls.mean))],
+  ];
+  const widths =
+    rows[0]?.map((_, index) => Math.max(...rows.map((row) => row[index]?.length ?? 0))) ?? [];
+  const separator = widths.map((width) => "─".repeat(width)).join("─┼─");
+  const table = rows.map((row) =>
+    row
+      .map((cell, index) => cell.padEnd(widths[index] ?? 0))
+      .join(" │ ")
+      .trimEnd(),
+  );
+  table.splice(1, 0, separator);
+  const failures = results.runs.filter((run) => run.failure !== null);
+  return [
+    `Benchmark: ${results.definition.name}`,
+    `Completed: ${results.runs.length}/${results.plannedRuns} (${results.status})`,
+    ...(unchecked
+      ? ["Correctness: not checked. Finished means execution completed without an error."]
+      : []),
+    "",
+    ...table,
+    "",
+    "Tokens include every request in a run. Failed runs contribute to these metrics too.",
+    "TTFT: first agent request start → first streamed text/reasoning/tool content; excludes worker startup.",
+    "Stream chunks may contain multiple tokens; TTFT is client-observed, not server token timing.",
+    "Throughput: total output tokens ÷ total task seconds, including network, tools, and validation.",
+    "Unknown values are n/a. Means/medians exclude missing values; throughput needs complete output usage for every run.",
+    ...(failures.length
+      ? [
+          "",
+          "Failed runs:",
+          ...failures.map((run) => {
+            const details = failureDetails(run);
+            return `- ${run.model} #${run.run}: ${run.failure}${details ? ` — ${details}` : ""}`;
+          }),
+        ]
+      : []),
+    "",
+    "Detailed per-run measurements and validator output: results.json.",
+  ].join("\n");
+}