pi-nebius 0.3.0

package/benchmarks/multi-file-feature/validation/check.test.mjs

@@ -0,0 +1,52 @@
+import assert from "node:assert/strict";
+import { resolve } from "node:path";
+import { test } from "node:test";
+import { pathToFileURL } from "node:url";
+
+const { handle } = await import(pathToFileURL(resolve("routes.mjs")));
+const store = await import(pathToFileURL(resolve("store.mjs")));
+const { serialize } = await import(pathToFileURL(resolve("serialize.mjs")));
+test("completion persists through store, serializer and HTTP routes", async () => {
+  assert.equal(typeof store.setCompleted, "function");
+  assert.deepEqual(serialize({ id: "x", title: "X", completed: true }), {
+    id: "x",
+    title: "X",
+    completed: true,
+  });
+  const created = await handle(
+    new Request("http://local/tasks", { method: "POST", body: JSON.stringify({ title: "First" }) }),
+  );
+  assert.equal(created.status, 201);
+  const task = await created.json();
+  assert.equal(task.completed, false);
+  for (const completed of [true, false]) {
+    const patched = await handle(
+      new Request(`http://local/tasks/${task.id}`, {
+        method: "PATCH",
+        body: JSON.stringify({ completed }),
+      }),
+    );
+    assert.equal(patched.status, 200);
+    assert.equal((await patched.json()).completed, completed);
+    const list = await (await handle(new Request("http://local/tasks"))).json();
+    assert.equal(list.find((item) => item.id === task.id).completed, completed);
+  }
+  for (const body of ['{"completed":"true"}', "{}", "broken-json"]) {
+    assert.equal(
+      (await handle(new Request(`http://local/tasks/${task.id}`, { method: "PATCH", body })))
+        .status,
+      400,
+    );
+  }
+  assert.equal(
+    (
+      await handle(
+        new Request("http://local/tasks/missing", { method: "PATCH", body: '{"completed":true}' }),
+      )
+    ).status,
+    404,
+  );
+  assert.equal(store.setCompleted("missing", true), null);
+  assert.equal(store.setCompleted(task.id, true).completed, true);
+  assert.equal(store.list().find((item) => item.id === task.id).completed, true);
+});

package/benchmarks/refactor-module/benchmark.yaml

@@ -0,0 +1,12 @@
+schemaVersion: 1
+name: refactor-module
+task: |
+  Refactor invoice.mjs to remove the duplicated line-subtotal calculation.
+  Export a lineSubtotal(lines) helper and have both regularTotal and priorityTotal
+  call it. Keep all existing numeric behavior, discounts and shipping charges.
+  There should be a single reduce-based subtotal implementation. Do not change tests.
+validation:
+  - command: node
+    args: [--test, "{validation}/check.test.mjs"]
+timeout: 120
+validationTimeout: 15

package/benchmarks/refactor-module/fixture/invoice.mjs

@@ -0,0 +1,8 @@
+export function regularTotal(lines, discount = 0) {
+  const subtotal = lines.reduce((sum, line) => sum + line.price * line.quantity, 0);
+  return subtotal * (1 - discount) + 5;
+}
+export function priorityTotal(lines, discount = 0) {
+  const subtotal = lines.reduce((sum, line) => sum + line.price * line.quantity, 0);
+  return subtotal * (1 - discount) + 15;
+}

package/benchmarks/refactor-module/fixture/invoice.test.mjs

@@ -0,0 +1,9 @@
+import assert from "node:assert/strict";
+import { test } from "node:test";
+import { priorityTotal, regularTotal } from "./invoice.mjs";
+
+test("totals", () => {
+  const lines = [{ price: 10, quantity: 2 }];
+  assert.equal(regularTotal(lines, 0.1), 23);
+  assert.equal(priorityTotal(lines, 0.1), 33);
+});

package/benchmarks/refactor-module/fixture/package.json

@@ -0,0 +1,8 @@
+{
+  "name": "refactor-fixture",
+  "private": true,
+  "type": "module",
+  "scripts": {
+    "test": "node --test"
+  }
+}

package/benchmarks/refactor-module/validation/check.test.mjs

@@ -0,0 +1,36 @@
+import assert from "node:assert/strict";
+import { readFile } from "node:fs/promises";
+import { resolve } from "node:path";
+import { test } from "node:test";
+import { pathToFileURL } from "node:url";
+
+const invoice = await import(pathToFileURL(resolve("invoice.mjs")));
+test("shared subtotal and preserved behavior", async () => {
+  assert.equal(typeof invoice.lineSubtotal, "function");
+  for (const lines of [
+    [],
+    [{ price: 10, quantity: 2 }],
+    [
+      { price: 1.5, quantity: 3 },
+      { price: 20, quantity: 2 },
+    ],
+  ]) {
+    const before = JSON.stringify(lines);
+    const subtotal = lines.reduce((total, line) => total + line.price * line.quantity, 0);
+    assert.equal(invoice.lineSubtotal(lines), subtotal);
+    for (const discount of [0, 0.1, 1]) {
+      assert.equal(invoice.regularTotal(lines, discount), subtotal * (1 - discount) + 5);
+      assert.equal(invoice.priorityTotal(lines, discount), subtotal * (1 - discount) + 15);
+    }
+    assert.equal(JSON.stringify(lines), before);
+  }
+  const source = await readFile("invoice.mjs", "utf8");
+  assert.equal(
+    (source.match(/\.reduce\s*\(/g) ?? []).length,
+    1,
+    "One shared reduce implementation",
+  );
+  for (const name of ["regularTotal", "priorityTotal"]) {
+    assert.match(invoice[name].toString(), /lineSubtotal\s*\(/, `${name} must call the helper`);
+  }
+});

package/dist/benchmark/cli.js

@@ -0,0 +1,112 @@
+#!/usr/bin/env node
+import { randomUUID } from "node:crypto";
+import { mkdtemp, rm } from "node:fs/promises";
+import { tmpdir } from "node:os";
+import { join, resolve } from "node:path";
+import { parseArgs } from "node:util";
+import { getAgentDir } from "@earendil-works/pi-coding-agent";
+import { discoverModels, MISSING_KEY } from "../discovery.js";
+import { applyModelSettings, loadSettings, settingsPath } from "../model-settings.js";
+import { loadDefinition, positive } from "./definition.js";
+import { redactor } from "./instrumentation.js";
+import { terminalReport } from "./report.js";
+import { runBenchmark } from "./runner.js";
+const HELP = `Usage: pi-nebius benchmark --benchmark DIRECTORY --models ID,ID [options]
+
+  --benchmark PATH   Directory with benchmark.yaml, or YAML/JSON definition file
+  --models IDS       Comma-separated, exact Nebius model IDs (no automatic selection)
+  --runs N           Repetitions per model (default: 1)
+  --timeout SECONDS  Per-agent deadline, including worker/session startup
+  --output PATH      New result directory (must not already exist)
+  --concurrency N    v1 supports 1 only: identical Pi prompts without cwd rewriting
+  --help             Show this help
+
+Each run receives a fresh fixture copy and an isolated Pi configuration.
+Pi tools have normal host access: filesystem copies are not a security sandbox.
+Use a disposable machine/container for untrusted fixtures or model-generated shell commands.
+No API key or prompt/response/tool content is retained in request traces.
+`;
+async function main() {
+    const args = process.argv.slice(2);
+    if (args[0] === "--help" || args[0] === "-h" || args.length === 0) {
+        console.log(HELP);
+        return;
+    }
+    if (args.shift() !== "benchmark")
+        throw new Error("Expected the benchmark subcommand. Use --help.");
+    const { values } = parseArgs({
+        args,
+        strict: true,
+        options: {
+            benchmark: { type: "string" },
+            models: { type: "string" },
+            runs: { type: "string" },
+            timeout: { type: "string" },
+            output: { type: "string" },
+            concurrency: { type: "string" },
+            help: { type: "boolean", short: "h" },
+        },
+    });
+    if (values.help) {
+        console.log(HELP);
+        return;
+    }
+    if (!values.benchmark || !values.models)
+        throw new Error("--benchmark and --models are required");
+    const runs = positive(Number(values.runs ?? 1), "runs", 1000);
+    const concurrency = positive(Number(values.concurrency ?? 1), "concurrency", 1);
+    const ids = values.models.split(",").map((id) => id.trim());
+    if (ids.some((id) => !id) || new Set(ids).size !== ids.length)
+        throw new Error("--models must contain nonempty, unique exact IDs");
+    if (runs * ids.length > 10000)
+        throw new Error("At most 10,000 runs per invocation");
+    const { definition, directory } = await loadDefinition(values.benchmark);
+    if (values.timeout)
+        definition.timeout = positive(Number(values.timeout), "timeout");
+    const apiKey = process.env.NEBIUS_API_KEY?.trim();
+    if (!apiKey)
+        throw new Error(MISSING_KEY);
+    const cache = await mkdtemp(join(tmpdir(), "pi-nebius-benchmark-discovery-"));
+    const discovered = await discoverModels({ apiKey, agentDir: cache, force: true }).finally(() => rm(cache, { recursive: true, force: true }));
+    if (discovered.warning)
+        process.stderr.write(`${discovered.warning}\n`);
+    const modelSettings = await loadSettings(settingsPath(getAgentDir()));
+    const models = ids.map((id) => {
+        const model = discovered.models.find((candidate) => candidate.id === id);
+        if (!model)
+            throw new Error(`Requested model was not discovered: ${id}. Check its exact ID and your access.`);
+        return applyModelSettings(model, modelSettings[id]);
+    });
+    const output = resolve(values.output ??
+        join("benchmark-results", `${new Date().toISOString().replaceAll(":", "-")}-${randomUUID().slice(0, 8)}`));
+    const controller = new AbortController();
+    const abort = () => controller.abort();
+    process.once("SIGINT", abort);
+    process.once("SIGTERM", abort);
+    try {
+        const results = await runBenchmark({
+            definition,
+            directory,
+            models,
+            modelSettings,
+            runs,
+            concurrency,
+            output,
+            apiKey,
+            signal: controller.signal,
+            onRun: (run) => process.stderr.write(`${run.model} #${run.run}: ${run.success ? "PASS" : run.failure}\n`),
+        });
+        console.log(terminalReport(results));
+        console.log(`Results: ${join(output, "results.json")}`);
+        process.exitCode =
+            results.status === "cancelled" ? 130 : results.runs.every((run) => run.success) ? 0 : 1;
+    }
+    finally {
+        process.removeListener("SIGINT", abort);
+        process.removeListener("SIGTERM", abort);
+    }
+}
+void main().catch((error) => {
+    console.error(redactor([process.env.NEBIUS_API_KEY ?? ""])(String(error)));
+    process.exitCode = 2;
+});

package/dist/benchmark/command.js

@@ -0,0 +1,194 @@
+import { randomUUID } from "node:crypto";
+import { mkdir, mkdtemp, readFile, realpath, rm } from "node:fs/promises";
+import { findPackageJSON } from "node:module";
+import { tmpdir } from "node:os";
+import { dirname, join } from "node:path";
+import { fileURLToPath, pathToFileURL } from "node:url";
+import { parseArgs } from "node:util";
+import { MISSING_KEY } from "../discovery.js";
+import { applyModelSettings } from "../model-settings.js";
+import { loadDefinition, positive } from "./definition.js";
+import { redactor } from "./instrumentation.js";
+import { snapshotProject } from "./project.js";
+import { terminalReport } from "./report.js";
+import { runBenchmark } from "./runner.js";
+const tasks = ["fix-auth-bug", "add-api-endpoint", "refactor-module", "multi-file-feature"];
+const help = `Usage: /nebius-benchmark [--models ID,ID] [--runs N]
+Enter your task prompt in the editor. Each run gets a fresh copy of the current project.
+Without --models, uses the selected Nebius model. Default: 1 run per model.
+Optional: --task NAME uses a bundled task instead (${tasks.join(", ")}).
+/nebius-benchmark cancel stops the active benchmark.
+Results appear here and in benchmark-results/. Custom prompts have no correctness check.
+Copies exclude Git-ignored files, dependencies, build output, and known credential files.
+Runs use paid inference and tools with normal host access; copies are not a security sandbox.`;
+export async function hostPiEntry() {
+    // The running Pi CLI can be a symlink into a global installation.
+    const base = process.argv[1]
+        ? pathToFileURL(await realpath(process.argv[1])).href
+        : import.meta.url;
+    const manifestPath = findPackageJSON("@earendil-works/pi-coding-agent", base);
+    if (!manifestPath)
+        throw new Error("Cannot locate the host Pi SDK. Use the Node.js Pi installation.");
+    const manifest = JSON.parse(await readFile(manifestPath, "utf8"));
+    return join(dirname(manifestPath), manifest.main);
+}
+export function registerBenchmarkCommand(pi, getSettings = () => ({})) {
+    let starting = false;
+    let active;
+    const show = (content) => pi.sendMessage({ customType: "nebius-benchmark", content, display: true }, { triggerTurn: false });
+    pi.registerCommand("nebius-benchmark", {
+        description: "Compare models on your prompt: --models ID,ID --runs N; help or cancel",
+        handler: async (args, ctx) => {
+            const tokens = args.trim().split(/\s+/).filter(Boolean);
+            const action = tokens[0];
+            if (action === "help" || action === "list") {
+                show(help);
+                return;
+            }
+            if (action === "cancel") {
+                if (active) {
+                    active.controller.abort();
+                    ctx.ui.notify("Stopping benchmark…", "info");
+                }
+                else
+                    ctx.ui.notify("No benchmark is running.", "info");
+                return;
+            }
+            if (active || starting) {
+                ctx.ui.notify("A benchmark is already running. Use /nebius-benchmark cancel.", "warning");
+                return;
+            }
+            const key = process.env.NEBIUS_API_KEY?.trim();
+            const redact = redactor([key ?? ""]);
+            starting = true;
+            let scratch;
+            try {
+                const { values } = parseArgs({
+                    args: tokens,
+                    options: {
+                        models: { type: "string" },
+                        runs: { type: "string" },
+                        task: { type: "string" },
+                    },
+                });
+                const runs = positive(Number(values.runs ?? 1), "runs", 1000);
+                if (values.task && !tasks.includes(values.task))
+                    throw new Error(help);
+                let selected = ctx.model ? [ctx.model] : [];
+                if (values.models !== undefined) {
+                    const ids = values.models.split(",");
+                    if (ids.some((id) => !id) || new Set(ids).size !== ids.length)
+                        throw new Error("--models requires unique, comma-separated model IDs.");
+                    const available = ctx.modelRegistry.getAll();
+                    selected = ids.map((id) => {
+                        const model = available.find((candidate) => candidate.provider === "nebius" && candidate.id === id);
+                        if (!model)
+                            throw new Error(`Unknown Nebius model: ${id}. Run /nebius-refresh or check /model.`);
+                        return model;
+                    });
+                }
+                if (!selected.length ||
+                    selected.some((model) => model.provider !== "nebius" || model.api !== "openai-completions"))
+                    throw new Error("Select a Nebius model with /model first.");
+                const total = runs * selected.length;
+                if (total > 10000)
+                    throw new Error("At most 10,000 runs per benchmark.");
+                if (!key)
+                    throw new Error(MISSING_KEY);
+                const piEntry = await hostPiEntry();
+                let definition;
+                let directory;
+                const task = values.task ?? "custom-prompt";
+                if (values.task) {
+                    const root = fileURLToPath(new URL("../../", import.meta.url));
+                    ({ definition, directory } = await loadDefinition(join(root, "benchmarks", task)));
+                }
+                else {
+                    if (!ctx.hasUI)
+                        throw new Error("Custom prompts need interactive Pi. Use --task for a bundled task.");
+                    const prompt = await ctx.ui.editor("Benchmark task — what should each model do?");
+                    if (!prompt?.trim())
+                        return;
+                    scratch = await mkdtemp(join(tmpdir(), "pi-nebius-project-"));
+                    directory = scratch;
+                    const copied = await snapshotProject(ctx.cwd, join(directory, "fixture"));
+                    await mkdir(join(directory, "validation"));
+                    definition = {
+                        schemaVersion: 1,
+                        name: task,
+                        task: prompt.trim(),
+                        fixture: "fixture",
+                        validationDirectory: "validation",
+                        validation: [],
+                        validationMode: "none",
+                        setup: [],
+                        timeout: 600,
+                        validationTimeout: 60,
+                        tools: ["read", "bash", "edit", "write", "grep", "find", "ls"],
+                    };
+                    ctx.ui.notify(`Copied ${copied} project files for comparison. Dependencies and ignored files are excluded.`, "info");
+                }
+                // Snapshot selection; changing the interactive model does not change an active run.
+                const modelSettings = structuredClone(getSettings());
+                const models = structuredClone(selected).map((model) => applyModelSettings(model, modelSettings[model.id]));
+                let completed = 0;
+                const output = join(ctx.cwd, "benchmark-results", `${task}-${randomUUID()}`);
+                const controller = new AbortController();
+                show(`Starting ${task}: ${models.map((model) => model.id).join(", ")}, ${runs} run(s) each. Uses paid inference; tools have normal host access.\nUse /nebius-benchmark cancel to stop.`);
+                ctx.ui.setStatus("nebius-benchmark", `Benchmark: ${task} (0/${total})`);
+                const snapshot = scratch;
+                scratch = undefined; // The background run owns cleanup from here.
+                const done = runBenchmark({
+                    definition,
+                    directory,
+                    models,
+                    modelSettings,
+                    runs,
+                    concurrency: 1,
+                    output,
+                    apiKey: key,
+                    signal: controller.signal,
+                    piEntry,
+                    workerPath: fileURLToPath(new URL(import.meta.url.endsWith(".ts") ? "./host-worker.ts" : "./host-worker.js", import.meta.url)),
+                    workerArgs: [piEntry],
+                    onRun: (run) => {
+                        ctx.ui.setStatus("nebius-benchmark", `Benchmark: ${task} (${++completed}/${total})`);
+                        ctx.ui.notify(`${run.model} #${run.run}: ${run.failure ?? (run.validation.checked ? "PASS" : "FINISHED (not validated)")}`, "info");
+                    },
+                })
+                    .then((results) => {
+                    show(redact(`${terminalReport(results)}\nDetails: ${join(output, "results.json")}`));
+                })
+                    .catch((error) => {
+                    show(redact(`Benchmark failed: ${String(error)}`));
+                })
+                    .finally(async () => {
+                    try {
+                        if (snapshot)
+                            await rm(snapshot, { recursive: true, force: true });
+                    }
+                    catch (error) {
+                        ctx.ui.notify(redact(`Could not remove temporary snapshot: ${String(error)}`), "warning");
+                    }
+                    finally {
+                        active = undefined;
+                        ctx.ui.setStatus("nebius-benchmark", undefined);
+                    }
+                });
+                active = { controller, done };
+            }
+            catch (error) {
+                ctx.ui.notify(redact(String(error)), "error");
+            }
+            finally {
+                if (scratch)
+                    await rm(scratch, { recursive: true, force: true });
+                starting = false;
+            }
+        },
+    });
+    pi.on("session_shutdown", async () => {
+        active?.controller.abort();
+        await active?.done;
+    });
+}

package/dist/benchmark/definition.js

@@ -0,0 +1,109 @@
+import { readFile, realpath } from "node:fs/promises";
+import { dirname, isAbsolute, join, relative, resolve, sep } from "node:path";
+import { parse } from "yaml";
+import { isRecord } from "../models.js";
+export function positive(value, name, maximum = 86400) {
+    if (typeof value !== "number" || !Number.isSafeInteger(value) || value < 1 || value > maximum)
+        throw new Error(`${name} must be an integer between 1 and ${maximum}`);
+    return value;
+}
+function text(value, name) {
+    if (typeof value !== "string" || !value.trim())
+        throw new Error(`${name} must be nonempty text`);
+    return value;
+}
+function fields(value, allowed) {
+    for (const key of Object.keys(value))
+        if (!allowed.includes(key))
+            throw new Error(`Unknown field: ${key}`);
+}
+function commands(value, name) {
+    if (!Array.isArray(value))
+        throw new Error(`${name} must be an array`);
+    return value.map((item) => {
+        if (typeof item === "string")
+            return { command: "/bin/sh", args: ["-c", text(item, name)] };
+        if (!isRecord(item))
+            throw new Error(`Invalid ${name} command`);
+        fields(item, ["command", "args"]);
+        if (item.args !== undefined &&
+            (!Array.isArray(item.args) || item.args.some((arg) => typeof arg !== "string")))
+            throw new Error(`${name} args must be strings`);
+        return { command: text(item.command, "command"), args: (item.args ?? []) };
+    });
+}
+function localPath(value, fallback) {
+    const path = value === undefined ? fallback : text(value, "path");
+    if (isAbsolute(path) || path.split(/[\\/]/).includes(".."))
+        throw new Error("Benchmark paths must be relative and inside its directory");
+    return path;
+}
+export function parseDefinition(source) {
+    const value = parse(source, { maxAliasCount: 0, uniqueKeys: true });
+    if (!isRecord(value))
+        throw new Error("Benchmark definition must be an object");
+    fields(value, [
+        "schemaVersion",
+        "name",
+        "task",
+        "fixture",
+        "validationDirectory",
+        "validation",
+        "setup",
+        "timeout",
+        "validationTimeout",
+        "tools",
+        "systemPrompt",
+    ]);
+    if (value.schemaVersion !== undefined && value.schemaVersion !== 1)
+        throw new Error("Unsupported benchmark schemaVersion");
+    const tools = value.tools ?? ["read", "bash", "edit", "write"];
+    const available = ["read", "bash", "edit", "write", "grep", "find", "ls"];
+    if (!Array.isArray(tools) ||
+        !tools.length ||
+        tools.some((tool) => !available.includes(tool)) ||
+        new Set(tools).size !== tools.length)
+        throw new Error("tools must be a nonempty, unique list of Pi built-in tools");
+    const validation = commands(value.validation, "validation");
+    if (!validation.length)
+        throw new Error("At least one validation command is required");
+    return {
+        schemaVersion: 1,
+        name: text(value.name, "name"),
+        task: text(value.task, "task"),
+        fixture: localPath(value.fixture, "fixture"),
+        validationDirectory: localPath(value.validationDirectory, "validation"),
+        validation,
+        setup: commands(value.setup ?? [], "setup"),
+        timeout: positive(value.timeout ?? 600, "timeout"),
+        validationTimeout: positive(value.validationTimeout ?? 60, "validationTimeout"),
+        tools,
+        ...(value.systemPrompt === undefined
+            ? {}
+            : { systemPrompt: text(value.systemPrompt, "systemPrompt") }),
+    };
+}
+export async function loadDefinition(path) {
+    const definitionFile = path.endsWith(".yaml") || path.endsWith(".yml") || path.endsWith(".json")
+        ? resolve(path)
+        : resolve(path, "benchmark.yaml");
+    const source = await readFile(definitionFile, "utf8");
+    if (Buffer.byteLength(source) > 1024 * 1024)
+        throw new Error("Definition exceeds 1 MiB");
+    const definition = parseDefinition(source);
+    const directory = await realpath(dirname(definitionFile));
+    const directories = [];
+    for (const child of [definition.fixture, definition.validationDirectory]) {
+        const actual = await realpath(join(directory, child));
+        directories.push(actual);
+        const rel = relative(directory, actual);
+        if (!rel || rel.startsWith("..") || isAbsolute(rel))
+            throw new Error("Fixture/validation must be separate directories inside the benchmark");
+    }
+    const [fixture, validators] = directories;
+    if (fixture &&
+        validators &&
+        [relative(fixture, validators), relative(validators, fixture)].some((path) => !isAbsolute(path) && path !== ".." && !path.startsWith(`..${sep}`)))
+        throw new Error("Fixture and validation directories must be separate and must not overlap");
+    return { definition, directory, source };
+}

package/dist/benchmark/host-worker.js

@@ -0,0 +1,14 @@
+import { registerHooks } from "node:module";
+import { pathToFileURL } from "node:url";
+// Reuse the installed host's SDK; Git installs intentionally omit Pi dev dependencies.
+const hostEntry = process.argv[2];
+if (!hostEntry)
+    throw new Error("Missing host Pi entry");
+registerHooks({
+    resolve(specifier, context, nextResolve) {
+        if (specifier.startsWith("@earendil-works/"))
+            return nextResolve(specifier, { ...context, parentURL: pathToFileURL(hostEntry).href });
+        return nextResolve(specifier, context);
+    },
+});
+await import("./worker.js");