rad-experiment 0.1.0

package/src/__tests__/fixtures.ts

@@ -0,0 +1,119 @@
+// Shared test data builders. Each returns a fresh object with sensible defaults.
+
+import type { Experiment, Measurement, MetricValue, Op, Reproduction } from "../types.js";
+
+export const SAMPLE_OID = "7554439853c68bc7ec063d6a7fcdc80f89e5b1fb";
+export const SAMPLE_AUTHOR_KEY = "z6Mkgoz7YreoB5v1jLvNCyX3MXLaza28Gs6d6yu3ABXE3cdY";
+
+export function makeMeasurement(overrides?: Partial<Measurement>): Measurement {
+  return { n: 5, medianX1000: 59340, stdX1000: 1200, ...overrides };
+}
+
+export function makeMetricValue(overrides?: Partial<MetricValue>): MetricValue {
+  return {
+    name: "binary_size",
+    unit: "bytes",
+    baseline: { n: 1, medianX1000: 1000000, stdX1000: 0 },
+    candidate: { n: 1, medianX1000: 950000, stdX1000: 0 },
+    deltaPctX100: -500,
+    regressed: false,
+    ...overrides,
+  };
+}
+
+/** Build a raw publish action with snake_case fields (as stored in COBs). */
+export function makeRawPublishAction(overrides?: Record<string, unknown>): Record<string, unknown> {
+  return {
+    type: "publish",
+    description: "SIMD vectorization in parser loop",
+    base: SAMPLE_OID,
+    oid: SAMPLE_OID,
+    metric_name: "wall_time",
+    metric_unit: "ms",
+    direction: "lower_is_better",
+    runner_class: "arm64",
+    os: "linux",
+    cpu: "apple-m2",
+    baseline: makeMeasurement(),
+    candidate: makeMeasurement({ n: 5, medianX1000: 45200, stdX1000: 950 }),
+    delta_pct_x100: 2378,
+    build_ok: true,
+    tests_ok: true,
+    sanitizers_ok: false,
+    agent_system: "claude-code",
+    agent_model: "claude-opus-4-6",
+    ...overrides,
+  };
+}
+
+/** Build a raw reproduce action with snake_case fields. */
+export function makeRawReproduceAction(overrides?: Record<string, unknown>): Record<string, unknown> {
+  return {
+    type: "reproduce",
+    verdict: "confirmed",
+    runner_class: "amd64",
+    baseline: makeMeasurement({ n: 10, medianX1000: 59000, stdX1000: 800 }),
+    candidate: makeMeasurement({ n: 10, medianX1000: 45500, stdX1000: 600 }),
+    delta_pct_x100: 2288,
+    build_ok: true,
+    tests_ok: true,
+    notes: "Reproduced on CI",
+    ...overrides,
+  };
+}
+
+/** Build a minimal valid Op containing a publish action. */
+export function makeOp(overrides?: Partial<Op> & { actions?: Record<string, unknown>[] }): Op {
+  const { actions, ...rest } = overrides ?? {};
+  return {
+    id: "aaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaaa",
+    actions: (actions ?? [makeRawPublishAction()]) as Op["actions"],
+    author: SAMPLE_AUTHOR_KEY,
+    timestamp: 1711300000000,
+    parents: [],
+    related: [],
+    identity: null,
+    manifest: { typeName: "cc.experiment", version: 1 },
+    ...rest,
+  };
+}
+
+export function makeReproduction(overrides?: Partial<Reproduction>): Reproduction {
+  return {
+    verdict: "confirmed",
+    runnerClass: "amd64",
+    baseline: makeMeasurement({ n: 10, medianX1000: 59000, stdX1000: 800 }),
+    candidate: makeMeasurement({ n: 10, medianX1000: 45500, stdX1000: 600 }),
+    deltaPctX100: 2288,
+    author: { id: `did:key:${SAMPLE_AUTHOR_KEY}` },
+    timestamp: 1711300001000,
+    ...overrides,
+  };
+}
+
+/** Build a camelCase Experiment state (as produced by the helper). */
+export function makeExperiment(overrides?: Partial<Experiment>): Experiment {
+  return {
+    description: "SIMD vectorization in parser loop",
+    base: SAMPLE_OID,
+    oid: SAMPLE_OID,
+    metricName: "wall_time",
+    metricUnit: "ms",
+    direction: "lower_is_better",
+    runnerClass: "arm64",
+    os: "linux",
+    cpu: "apple-m2",
+    baseline: makeMeasurement(),
+    candidate: makeMeasurement({ n: 5, medianX1000: 45200, stdX1000: 950 }),
+    deltaPctX100: 2378,
+    buildOk: true,
+    testsOk: true,
+    sanitizersOk: false,
+    agentSystem: "claude-code",
+    agentModel: "claude-opus-4-6",
+    reproductions: [],
+    author: { id: `did:key:${SAMPLE_AUTHOR_KEY}` },
+    createdAt: 1711300000000,
+    ...overrides,
+  };
+}

package/src/__tests__/format.test.ts

@@ -0,0 +1,55 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+import { deltaDisplay, measurementDisplay, shortId, confirmedCount } from "../cli/format.js";
+import { makeExperiment, makeReproduction } from "./fixtures.js";
+
+describe("deltaDisplay", () => {
+  it("formats positive value", () => assert.equal(deltaDisplay(547), "+5.47%"));
+  it("formats zero", () => assert.equal(deltaDisplay(0), "+0.00%"));
+  it("formats negative value", () => assert.equal(deltaDisplay(-78), "-0.78%"));
+  it("formats single-digit fractional", () => assert.equal(deltaDisplay(5), "+0.05%"));
+  it("formats exact percent", () => assert.equal(deltaDisplay(1000), "+10.00%"));
+  it("formats large value", () => assert.equal(deltaDisplay(23780), "+237.80%"));
+});
+
+describe("measurementDisplay", () => {
+  it("formats normal value", () => {
+    assert.equal(measurementDisplay({ n: 10, medianX1000: 59340, stdX1000: 120 }, "ms"), "59.340 ms");
+  });
+  it("formats sub-unit", () => {
+    assert.equal(measurementDisplay({ n: 5, medianX1000: 500, stdX1000: 10 }, "ms"), "0.500 ms");
+  });
+  it("formats zero", () => {
+    assert.equal(measurementDisplay({ n: 1, medianX1000: 0, stdX1000: 0 }, "s"), "0.0 s");
+  });
+});
+
+describe("shortId", () => {
+  it("truncates to 7 chars", () => assert.equal(shortId("abcdef1234567890abcd"), "abcdef1"));
+  it("handles short input", () => assert.equal(shortId("abc"), "abc"));
+});
+
+describe("confirmedCount", () => {
+  it("counts confirmed reproductions", () => {
+    const exp = makeExperiment({
+      reproductions: [
+        makeReproduction({ verdict: "confirmed" }),
+        makeReproduction({ verdict: "failed" }),
+        makeReproduction({ verdict: "confirmed" }),
+      ],
+    });
+    assert.equal(confirmedCount(exp), 2);
+  });
+
+  it("returns 0 when none confirmed", () => {
+    const exp = makeExperiment({
+      reproductions: [makeReproduction({ verdict: "inconclusive" })],
+    });
+    assert.equal(confirmedCount(exp), 0);
+  });
+
+  it("returns 0 for empty reproductions", () => {
+    assert.equal(confirmedCount(makeExperiment()), 0);
+  });
+});

package/src/__tests__/golden/publish-action.json

@@ -0,0 +1,46 @@
+{
+  "agent_model": "claude-opus-4-6",
+  "agent_system": "claude-code",
+  "base": "7554439853c68bc7ec063d6a7fcdc80f89e5b1fb",
+  "baseline": {
+    "medianX1000": 59340,
+    "n": 5,
+    "stdX1000": 1200
+  },
+  "build_ok": true,
+  "candidate": {
+    "medianX1000": 45200,
+    "n": 5,
+    "stdX1000": 950
+  },
+  "cpu": "apple-m2",
+  "delta_pct_x100": 2378,
+  "description": "SIMD vectorization in parser loop",
+  "direction": "lower_is_better",
+  "metric_name": "wall_time",
+  "metric_unit": "ms",
+  "oid": "7554439853c68bc7ec063d6a7fcdc80f89e5b1fb",
+  "os": "linux",
+  "runner_class": "arm64",
+  "sanitizers_ok": false,
+  "secondary_metrics": [
+    {
+      "baseline": {
+        "medianX1000": 1000000,
+        "n": 1,
+        "stdX1000": 0
+      },
+      "candidate": {
+        "medianX1000": 950000,
+        "n": 1,
+        "stdX1000": 0
+      },
+      "deltaPctX100": -500,
+      "name": "binary_size",
+      "regressed": false,
+      "unit": "bytes"
+    }
+  ],
+  "tests_ok": true,
+  "type": "publish"
+}

package/src/__tests__/golden/publish-minimal.json

@@ -0,0 +1,25 @@
+{
+  "agent_model": "claude-opus-4-6",
+  "agent_system": "claude-code",
+  "base": "defd08dcff239ec2a3a57ec82f541d21210e98f9",
+  "baseline": {
+    "medianX1000": 50000,
+    "n": 10,
+    "stdX1000": 0
+  },
+  "build_ok": true,
+  "candidate": {
+    "medianX1000": 55000,
+    "n": 10,
+    "stdX1000": 0
+  },
+  "delta_pct_x100": 1000,
+  "direction": "higher_is_better",
+  "metric_name": "throughput",
+  "metric_unit": "ops/s",
+  "oid": "defd08dcff239ec2a3a57ec82f541d21210e98f9",
+  "runner_class": "amd64",
+  "sanitizers_ok": false,
+  "tests_ok": true,
+  "type": "publish"
+}

package/src/__tests__/golden/publish-with-samples.json

@@ -0,0 +1,38 @@
+{
+  "agent_model": "claude-opus-4-6",
+  "agent_system": "claude-code",
+  "base": "defd08dcff239ec2a3a57ec82f541d21210e98f9",
+  "baseline": {
+    "medianX1000": 59340,
+    "n": 3,
+    "samplesX1000": [
+      58000,
+      59340,
+      60680
+    ],
+    "stdX1000": 1200
+  },
+  "build_ok": true,
+  "candidate": {
+    "medianX1000": 45200,
+    "n": 3,
+    "samplesX1000": [
+      44500,
+      45200,
+      45900
+    ],
+    "stdX1000": 950
+  },
+  "cpu": "apple-m2",
+  "delta_pct_x100": 2378,
+  "description": "With per-run samples",
+  "direction": "lower_is_better",
+  "metric_name": "wall_time",
+  "metric_unit": "ms",
+  "oid": "defd08dcff239ec2a3a57ec82f541d21210e98f9",
+  "os": "linux",
+  "runner_class": "arm64",
+  "sanitizers_ok": false,
+  "tests_ok": true,
+  "type": "publish"
+}

package/src/__tests__/golden/reproduce-action.json

@@ -0,0 +1,19 @@
+{
+  "baseline": {
+    "medianX1000": 59000,
+    "n": 10,
+    "stdX1000": 800
+  },
+  "build_ok": true,
+  "candidate": {
+    "medianX1000": 45500,
+    "n": 10,
+    "stdX1000": 600
+  },
+  "delta_pct_x100": 2288,
+  "notes": "Reproduced on CI",
+  "runner_class": "amd64",
+  "tests_ok": true,
+  "type": "reproduce",
+  "verdict": "confirmed"
+}

package/src/__tests__/golden/reproduce-minimal.json

@@ -0,0 +1,18 @@
+{
+  "baseline": {
+    "medianX1000": 50500,
+    "n": 5,
+    "stdX1000": 0
+  },
+  "build_ok": true,
+  "candidate": {
+    "medianX1000": 50800,
+    "n": 5,
+    "stdX1000": 0
+  },
+  "delta_pct_x100": 59,
+  "runner_class": "arm64",
+  "tests_ok": true,
+  "type": "reproduce",
+  "verdict": "inconclusive"
+}

package/src/__tests__/helpers.test.ts

@@ -0,0 +1,138 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+import {
+  CliError,
+  requireArg,
+  requireInt,
+  optionalInt,
+  buildMeasurement,
+  parseSecondary,
+} from "../cli/helpers.js";
+
+describe("requireArg", () => {
+  it("returns string value", () => {
+    assert.equal(requireArg({ foo: "bar" }, "foo"), "bar");
+  });
+
+  it("throws CliError on missing key", () => {
+    assert.throws(() => requireArg({}, "foo"), CliError);
+  });
+
+  it("throws CliError on empty string", () => {
+    assert.throws(() => requireArg({ foo: "" }, "foo"), CliError);
+  });
+
+  it("throws CliError on null", () => {
+    assert.throws(() => requireArg({ foo: null }, "foo"), CliError);
+  });
+});
+
+describe("requireInt", () => {
+  it("parses integer string", () => {
+    assert.equal(requireInt({ n: "42" }, "n"), 42);
+  });
+
+  it("parses negative integer", () => {
+    assert.equal(requireInt({ n: "-7" }, "n"), -7);
+  });
+
+  it("throws CliError on non-integer", () => {
+    assert.throws(() => requireInt({ n: "abc" }, "n"), CliError);
+  });
+
+  it("throws CliError on missing", () => {
+    assert.throws(() => requireInt({}, "n"), CliError);
+  });
+});
+
+describe("optionalInt", () => {
+  it("returns default when missing", () => {
+    assert.equal(optionalInt({}, "x", 99), 99);
+  });
+
+  it("returns default when empty string", () => {
+    assert.equal(optionalInt({ x: "" }, "x", 99), 99);
+  });
+
+  it("parses when present", () => {
+    assert.equal(optionalInt({ x: "7" }, "x", 99), 7);
+  });
+
+  it("throws CliError on non-integer", () => {
+    assert.throws(() => optionalInt({ x: "abc" }, "x", 99), CliError);
+  });
+});
+
+describe("buildMeasurement", () => {
+  it("builds baseline measurement", () => {
+    const m = buildMeasurement(
+      { "baseline-median": "5000", "baseline-std": "100", "baseline-n": "10", "baseline-samples": "1000,2000,3000" },
+      "baseline",
+    );
+    assert.equal(m.n, 10);
+    assert.equal(m.medianX1000, 5000);
+    assert.equal(m.stdX1000, 100);
+    assert.deepEqual(m.samplesX1000, [1000, 2000, 3000]);
+  });
+
+  it("builds candidate measurement", () => {
+    const m = buildMeasurement(
+      { "candidate-median": "4000", "candidate-n": "5" },
+      "candidate",
+    );
+    assert.equal(m.n, 5);
+    assert.equal(m.medianX1000, 4000);
+    assert.equal(m.stdX1000, 0); // default
+  });
+
+  it("defaults std to 0", () => {
+    const m = buildMeasurement({ "baseline-median": "100", "baseline-n": "1" }, "baseline");
+    assert.equal(m.stdX1000, 0);
+  });
+
+  it("omits samplesX1000 when empty", () => {
+    const m = buildMeasurement(
+      { "baseline-median": "100", "baseline-n": "1", "baseline-samples": "" },
+      "baseline",
+    );
+    assert.equal("samplesX1000" in m, false);
+  });
+});
+
+describe("parseSecondary", () => {
+  it("parses 5-field format", () => {
+    const mv = parseSecondary("rss:KB:5000:4500:-1000");
+    assert.equal(mv.name, "rss");
+    assert.equal(mv.unit, "KB");
+    assert.equal(mv.baseline.medianX1000, 5000);
+    assert.equal(mv.candidate.medianX1000, 4500);
+    assert.equal(mv.deltaPctX100, -1000);
+    assert.equal(mv.regressed, false);
+    assert.equal(mv.baseline.n, 1);
+    assert.equal(mv.baseline.stdX1000, 0);
+  });
+
+  it("parses 6-field format with regressed=true", () => {
+    const mv = parseSecondary("mem:MB:41000:42000:243:true");
+    assert.equal(mv.regressed, true);
+  });
+
+  it("defaults regressed to false", () => {
+    const mv = parseSecondary("x:y:1:2:3");
+    assert.equal(mv.regressed, false);
+  });
+
+  it("throws CliError on too few fields", () => {
+    assert.throws(() => parseSecondary("a:b:c"), CliError);
+  });
+
+  it("throws CliError on non-integer numeric fields", () => {
+    assert.throws(() => parseSecondary("a:b:notnum:2:3"), CliError);
+  });
+
+  it("handles negative delta", () => {
+    const mv = parseSecondary("x:y:100:200:-500");
+    assert.equal(mv.deltaPctX100, -500);
+  });
+});

package/src/__tests__/integration.test.ts

@@ -0,0 +1,124 @@
+// End-to-end integration tests. Requires `rad` on PATH with a configured identity.
+// Creates a temp radicle repo, runs CLI commands, verifies output.
+
+import { test } from "node:test";
+import assert from "node:assert/strict";
+import { execFileSync } from "node:child_process";
+import { mkdtempSync, rmSync } from "node:fs";
+import { join, dirname } from "node:path";
+import { tmpdir } from "node:os";
+import { fileURLToPath } from "node:url";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const CLI = join(__dirname, "..", "rad-experiment.ts");
+const TSX = join(__dirname, "..", "..", "node_modules", ".bin", "tsx");
+
+function radExperiment(...args: string[]): string {
+  return execFileSync(TSX, [CLI, ...args], { encoding: "utf-8", timeout: 15000 });
+}
+
+function shell(cmd: string, args: string[], opts?: { cwd?: string; timeout?: number }): string {
+  return execFileSync(cmd, args, { encoding: "utf-8", ...opts });
+}
+
+function shellQuiet(cmd: string, args: string[], opts?: { cwd?: string; timeout?: number }): void {
+  execFileSync(cmd, args, { stdio: "inherit", ...opts });
+}
+
+function canRunIntegration(): boolean {
+  try {
+    shell("rad", ["self"]);
+    return true;
+  } catch {
+    return false;
+  }
+}
+
+test("integration: full CLI workflow", { skip: !canRunIntegration() && "rad not available" }, () => {
+  const tempDir = mkdtempSync(join(tmpdir(), "rad-exp-test-"));
+
+  try {
+    // Setup: init git + radicle repo (use inherited stdio — rad init fails with piped stdio)
+    // Unique commit message avoids radicle identity collision across test runs.
+    shellQuiet("git", ["init", "-b", "main"], { cwd: tempDir });
+    shellQuiet("git", ["commit", "--allow-empty", "-m", `init-${Date.now()}`], { cwd: tempDir });
+    shellQuiet("rad", ["init", "--name", `test-${Date.now()}`, "--description", "test", "--default-branch", "main", "--public"], {
+      cwd: tempDir,
+      timeout: 15000,
+    });
+    const commit = shell("git", ["rev-parse", "HEAD"], { cwd: tempDir }).trim();
+
+    // publish
+    const publishOutput = radExperiment(
+      "--repo", tempDir,
+      "publish",
+      "--base", commit, "--head", commit,
+      "--metric", "wall_time", "--unit", "ms",
+      "--direction", "lower_is_better",
+      "--runner", "arm64",
+      "--baseline-median", "59340", "--baseline-n", "5",
+      "--candidate-median", "45200", "--candidate-n", "5",
+      "--delta", "2378",
+      "-d", "Integration test experiment",
+    );
+    assert.match(publishOutput, /Experiment published: ([a-f0-9]{40})/);
+    assert.match(publishOutput, /\+23\.78%/);
+    const publishedId = publishOutput.match(/Experiment published: ([a-f0-9]{40})/)![1];
+
+    // list
+    const listOutput = radExperiment("--repo", tempDir, "list");
+    assert.match(listOutput, new RegExp(publishedId.slice(0, 7)));
+    assert.match(listOutput, /wall_time/);
+
+    // show (text)
+    const showOutput = radExperiment("--repo", tempDir, "show", publishedId);
+    assert.match(showOutput, /Integration test experiment/);
+    assert.match(showOutput, /wall_time \(ms\)/);
+    assert.match(showOutput, /59\.340 ms/);
+    assert.match(showOutput, /\+23\.78%/);
+
+    // show --json
+    const jsonOutput = radExperiment("--repo", tempDir, "show", "--json", publishedId);
+    const exp = JSON.parse(jsonOutput);
+    assert.equal(exp.metricName, "wall_time");
+    assert.equal(exp.deltaPctX100, 2378);
+    assert.ok(Array.isArray(exp.reproductions));
+
+    // reproduce
+    const reproOutput = radExperiment(
+      "--repo", tempDir,
+      "reproduce", publishedId,
+      "--verdict", "confirmed",
+      "--runner", "amd64",
+      "--baseline-median", "59000", "--baseline-n", "5",
+      "--candidate-median", "45500", "--candidate-n", "5",
+      "--delta", "2288",
+    );
+    assert.match(reproOutput, /Reproduction added/);
+
+    // verify reproduction in show --json
+    const afterReproJson = radExperiment("--repo", tempDir, "show", "--json", publishedId);
+    const expAfter = JSON.parse(afterReproJson);
+    assert.equal(expAfter.reproductions.length, 1);
+    assert.equal(expAfter.reproductions[0].verdict, "confirmed");
+
+    // list --reproduced
+    const reproList = radExperiment("--repo", tempDir, "list", "--reproduced");
+    assert.match(reproList, /verified/);
+
+    // list --unverified
+    const unverifiedList = radExperiment("--repo", tempDir, "list", "--unverified");
+    assert.match(unverifiedList, /No experiments found/);
+
+    // error path
+    assert.throws(
+      () => radExperiment("--repo", tempDir, "publish"),
+      (err: unknown) => {
+        const e = err as { status: number; stderr: Buffer };
+        return e.status !== 0 && e.stderr.toString().includes("missing required argument");
+      },
+    );
+  } finally {
+    try { rmSync(tempDir, { recursive: true }); } catch { /* ignore */ }
+  }
+});