rad-experiment 0.1.0

package/src/__tests__/serialization.test.ts

@@ -0,0 +1,175 @@
+// Golden-file tests: verify TS action builders produce JSON identical to the Rust binary.
+// The golden files in ./golden/ were generated by running the Rust rad-experiment CLI
+// and extracting the raw action JSON via `rad cob log --format json`.
+
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+import { readFileSync } from "node:fs";
+import { fileURLToPath } from "node:url";
+import { dirname, join } from "node:path";
+
+import { buildPublishAction, buildReproduceAction } from "../cob/actions.js";
+import { makeMeasurement, makeMetricValue } from "./fixtures.js";
+
+const __dirname = dirname(fileURLToPath(import.meta.url));
+const goldenDir = join(__dirname, "golden");
+
+function loadGolden(name: string): Record<string, unknown> {
+  return JSON.parse(readFileSync(join(goldenDir, name), "utf-8"));
+}
+
+/**
+ * Normalize for comparison: deep-sort all object keys, strip `parents`
+ * (only present in TS-created actions via external COB path).
+ */
+function normalize(obj: unknown): string {
+  return JSON.stringify(obj, (_, v) => {
+    if (v && typeof v === "object" && !Array.isArray(v)) {
+      const sorted: Record<string, unknown> = {};
+      for (const k of Object.keys(v).sort()) {
+        if (k !== "parents") sorted[k] = v[k];
+      }
+      return sorted;
+    }
+    return v;
+  });
+}
+
+// --- Publish: full (all optional fields present) ---
+
+describe("golden: publish-action (all fields)", () => {
+  const golden = loadGolden("publish-action.json");
+
+  it("matches Rust output", () => {
+    const action = buildPublishAction({
+      description: "SIMD vectorization in parser loop",
+      base: golden.base as string,
+      oid: golden.oid as string,
+      metricName: "wall_time",
+      metricUnit: "ms",
+      direction: "lower_is_better",
+      runnerClass: "arm64",
+      os: "linux",
+      cpu: "apple-m2",
+      baseline: makeMeasurement(),
+      candidate: makeMeasurement({ n: 5, medianX1000: 45200, stdX1000: 950 }),
+      deltaPctX100: 2378,
+      buildOk: true,
+      testsOk: true,
+      sanitizersOk: false,
+      agentSystem: "claude-code",
+      agentModel: "claude-opus-4-6",
+      secondaryMetrics: [makeMetricValue()],
+    });
+    assert.equal(normalize(action), normalize(golden));
+  });
+});
+
+// --- Publish: minimal (no description, no os/cpu, no secondary_metrics) ---
+
+describe("golden: publish-minimal (skip_serializing_if paths)", () => {
+  const golden = loadGolden("publish-minimal.json");
+
+  it("matches Rust output — omitted fields absent", () => {
+    const action = buildPublishAction({
+      base: golden.base as string,
+      oid: golden.oid as string,
+      metricName: "throughput",
+      metricUnit: "ops/s",
+      direction: "higher_is_better",
+      runnerClass: "amd64",
+      os: "",
+      cpu: "",
+      baseline: makeMeasurement({ n: 10, medianX1000: 50000, stdX1000: 0 }),
+      candidate: makeMeasurement({ n: 10, medianX1000: 55000, stdX1000: 0 }),
+      deltaPctX100: 1000,
+      buildOk: true,
+      testsOk: true,
+      sanitizersOk: false,
+      agentSystem: "claude-code",
+      agentModel: "claude-opus-4-6",
+      secondaryMetrics: [],
+    });
+    assert.equal(normalize(action), normalize(golden));
+  });
+
+  it("has no description, os, cpu, or secondary_metrics keys", () => {
+    assert.equal("description" in golden, false);
+    assert.equal("os" in golden, false);
+    assert.equal("cpu" in golden, false);
+    assert.equal("secondary_metrics" in golden, false);
+  });
+});
+
+// --- Publish: with samplesX1000 in Measurements ---
+
+describe("golden: publish-with-samples (nested Measurement samples)", () => {
+  const golden = loadGolden("publish-with-samples.json");
+
+  it("matches Rust output — samplesX1000 preserved", () => {
+    const action = buildPublishAction({
+      description: "With per-run samples",
+      base: golden.base as string,
+      oid: golden.oid as string,
+      metricName: "wall_time",
+      metricUnit: "ms",
+      direction: "lower_is_better",
+      runnerClass: "arm64",
+      os: "linux",
+      cpu: "apple-m2",
+      baseline: makeMeasurement({ n: 3, medianX1000: 59340, stdX1000: 1200, samplesX1000: [58000, 59340, 60680] }),
+      candidate: makeMeasurement({ n: 3, medianX1000: 45200, stdX1000: 950, samplesX1000: [44500, 45200, 45900] }),
+      deltaPctX100: 2378,
+      buildOk: true,
+      testsOk: true,
+      sanitizersOk: false,
+      agentSystem: "claude-code",
+      agentModel: "claude-opus-4-6",
+      secondaryMetrics: [],
+    });
+    assert.equal(normalize(action), normalize(golden));
+  });
+});
+
+// --- Reproduce: full (with notes) ---
+
+describe("golden: reproduce-action (with notes)", () => {
+  const golden = loadGolden("reproduce-action.json");
+
+  it("matches Rust output", () => {
+    const action = buildReproduceAction({
+      verdict: "confirmed",
+      runnerClass: "amd64",
+      baseline: makeMeasurement({ n: 10, medianX1000: 59000, stdX1000: 800 }),
+      candidate: makeMeasurement({ n: 10, medianX1000: 45500, stdX1000: 600 }),
+      deltaPctX100: 2288,
+      buildOk: true,
+      testsOk: true,
+      notes: "Reproduced on CI",
+    });
+    assert.equal(normalize(action), normalize(golden));
+  });
+});
+
+// --- Reproduce: minimal (no notes) ---
+
+describe("golden: reproduce-minimal (no notes)", () => {
+  const golden = loadGolden("reproduce-minimal.json");
+
+  it("matches Rust output — notes absent", () => {
+    const action = buildReproduceAction({
+      verdict: "inconclusive",
+      runnerClass: "arm64",
+      baseline: makeMeasurement({ n: 5, medianX1000: 50500, stdX1000: 0 }),
+      candidate: makeMeasurement({ n: 5, medianX1000: 50800, stdX1000: 0 }),
+      deltaPctX100: 59,
+      buildOk: true,
+      testsOk: true,
+    });
+    assert.equal(normalize(action), normalize(golden));
+  });
+
+  it("has no notes key", () => {
+    assert.equal("notes" in golden, false);
+  });
+});

package/src/__tests__/state.test.ts

@@ -0,0 +1,191 @@
+import { describe, it } from "node:test";
+import assert from "node:assert/strict";
+
+import { fromRoot, applyOp, handleOpMessage } from "../cob/state.js";
+import {
+  makeOp,
+  makeRawPublishAction,
+  makeRawReproduceAction,
+  makeExperiment,
+  SAMPLE_AUTHOR_KEY,
+} from "./fixtures.js";
+
+describe("fromRoot", () => {
+  it("creates Experiment from a publish op", () => {
+    const op = makeOp();
+    const exp = fromRoot(op);
+    assert.equal(exp.metricName, "wall_time");
+    assert.equal(exp.metricUnit, "ms");
+    assert.equal(exp.direction, "lower_is_better");
+    assert.equal(exp.runnerClass, "arm64");
+    assert.equal(exp.deltaPctX100, 2378);
+    assert.equal(exp.buildOk, true);
+    assert.equal(exp.agentSystem, "claude-code");
+    assert.deepEqual(exp.reproductions, []);
+  });
+
+  it("prefixes author with did:key:", () => {
+    const exp = fromRoot(makeOp({ author: "z6Mktest" }));
+    assert.equal(exp.author.id, "did:key:z6Mktest");
+  });
+
+  it("does not double-prefix did:key:", () => {
+    const exp = fromRoot(makeOp({ author: "did:key:z6Mktest" }));
+    assert.equal(exp.author.id, "did:key:z6Mktest");
+  });
+
+  it("preserves description when present", () => {
+    const exp = fromRoot(makeOp({ actions: [makeRawPublishAction({ description: "hello" })] }));
+    assert.equal(exp.description, "hello");
+  });
+
+  it("omits description when absent", () => {
+    const action = makeRawPublishAction();
+    delete action.description;
+    const exp = fromRoot(makeOp({ actions: [action] }));
+    assert.equal("description" in exp, false);
+  });
+
+  it("reads snake_case fields from Rust-created actions", () => {
+    const exp = fromRoot(makeOp({
+      actions: [makeRawPublishAction({ metric_name: "throughput", runner_class: "x86" })],
+    }));
+    assert.equal(exp.metricName, "throughput");
+    assert.equal(exp.runnerClass, "x86");
+  });
+
+  it("reads camelCase fields from legacy TS-created actions", () => {
+    const action = makeRawPublishAction();
+    // Replace snake_case with camelCase (simulate legacy format)
+    delete action.metric_name;
+    delete action.runner_class;
+    delete action.delta_pct_x100;
+    delete action.build_ok;
+    delete action.tests_ok;
+    delete action.sanitizers_ok;
+    delete action.agent_system;
+    delete action.agent_model;
+    (action as Record<string, unknown>).metricName = "throughput";
+    (action as Record<string, unknown>).runnerClass = "x86";
+    (action as Record<string, unknown>).deltaPctX100 = 500;
+    (action as Record<string, unknown>).buildOk = true;
+    (action as Record<string, unknown>).testsOk = true;
+    (action as Record<string, unknown>).sanitizersOk = false;
+    (action as Record<string, unknown>).agentSystem = "claude-code";
+    (action as Record<string, unknown>).agentModel = "claude-opus-4-6";
+
+    const exp = fromRoot(makeOp({ actions: [action] }));
+    assert.equal(exp.metricName, "throughput");
+    assert.equal(exp.runnerClass, "x86");
+    assert.equal(exp.deltaPctX100, 500);
+  });
+
+  it("propagates secondary_metrics when non-empty", () => {
+    const action = makeRawPublishAction({
+      secondary_metrics: [{ name: "rss", unit: "KB", baseline: { n: 1, medianX1000: 100, stdX1000: 0 }, candidate: { n: 1, medianX1000: 90, stdX1000: 0 }, deltaPctX100: -1000, regressed: false }],
+    });
+    const exp = fromRoot(makeOp({ actions: [action] }));
+    assert.equal(exp.secondaryMetrics?.length, 1);
+    assert.equal(exp.secondaryMetrics?.[0].name, "rss");
+  });
+
+  it("omits secondaryMetrics when empty", () => {
+    const action = makeRawPublishAction({ secondary_metrics: [] });
+    const exp = fromRoot(makeOp({ actions: [action] }));
+    assert.equal("secondaryMetrics" in exp, false);
+  });
+
+  it("applies remaining actions in root op", () => {
+    const publish = makeRawPublishAction();
+    const reproduce = makeRawReproduceAction();
+    const exp = fromRoot(makeOp({ actions: [publish, reproduce] }));
+    assert.equal(exp.reproductions.length, 1);
+    assert.equal(exp.reproductions[0].verdict, "confirmed");
+  });
+
+  it("throws on non-publish first action", () => {
+    assert.throws(
+      () => fromRoot(makeOp({ actions: [makeRawReproduceAction()] })),
+      { message: "the first action must be of type `publish`" },
+    );
+  });
+
+  it("throws on empty actions", () => {
+    assert.throws(
+      () => fromRoot(makeOp({ actions: [] })),
+      { message: "the first action must be of type `publish`" },
+    );
+  });
+});
+
+describe("applyOp", () => {
+  it("adds reproduction to experiment", () => {
+    const exp = makeExperiment();
+    applyOp(exp, makeOp({ actions: [makeRawReproduceAction()] }));
+    assert.equal(exp.reproductions.length, 1);
+    assert.equal(exp.reproductions[0].verdict, "confirmed");
+    assert.equal(exp.reproductions[0].runnerClass, "amd64");
+  });
+
+  it("ignores publish actions (no-op)", () => {
+    const exp = makeExperiment();
+    applyOp(exp, makeOp({ actions: [makeRawPublishAction()] }));
+    assert.equal(exp.reproductions.length, 0);
+  });
+
+  it("applies multiple actions in one op", () => {
+    const exp = makeExperiment();
+    applyOp(exp, makeOp({
+      actions: [
+        makeRawReproduceAction({ verdict: "confirmed" }),
+        makeRawReproduceAction({ verdict: "failed" }),
+      ],
+    }));
+    assert.equal(exp.reproductions.length, 2);
+    assert.equal(exp.reproductions[0].verdict, "confirmed");
+    assert.equal(exp.reproductions[1].verdict, "failed");
+  });
+});
+
+describe("handleOpMessage", () => {
+  it("creates experiment from empty value (root)", () => {
+    const result = handleOpMessage({
+      value: {},
+      op: makeOp(),
+      concurrent: [],
+    });
+    assert.equal(result.metricName, "wall_time");
+    assert.deepEqual(result.reproductions, []);
+  });
+
+  it("creates experiment from null value", () => {
+    const result = handleOpMessage({
+      value: null as unknown as Record<string, unknown>,
+      op: makeOp(),
+      concurrent: [],
+    });
+    assert.equal(result.metricName, "wall_time");
+  });
+
+  it("applies op to existing state", () => {
+    const existing = makeExperiment() as unknown as Record<string, unknown>;
+    const result = handleOpMessage({
+      value: existing,
+      op: makeOp({ actions: [makeRawReproduceAction()] }),
+      concurrent: [],
+    });
+    assert.equal(result.reproductions.length, 1);
+  });
+
+  it("applies concurrent ops", () => {
+    const result = handleOpMessage({
+      value: {},
+      op: makeOp(),
+      concurrent: [
+        makeOp({ actions: [makeRawReproduceAction({ verdict: "confirmed" })] }),
+        makeOp({ actions: [makeRawReproduceAction({ verdict: "failed" })] }),
+      ],
+    });
+    assert.equal(result.reproductions.length, 2);
+  });
+});

package/src/cli/commands/list.ts

@@ -0,0 +1,45 @@
+import { parseArgs } from "node:util";
+import { deltaDisplay, shortId, confirmedCount } from "../format.js";
+import { getRepoId, cobList, cobShow } from "../rad.js";
+import type { Experiment } from "../../types.js";
+
+export function cmdList(args: string[]): void {
+  const { values } = parseArgs({
+    args,
+    options: {
+      repo: { type: "string", short: "r" },
+      reproduced: { type: "boolean", default: false },
+      unverified: { type: "boolean", default: false },
+    },
+    strict: true,
+  });
+
+  const rid = getRepoId(values.repo as string | undefined);
+  const ids = cobList(rid);
+
+  const experiments: { id: string; exp: Experiment }[] = [];
+  for (const id of ids) {
+    experiments.push({ id, exp: cobShow(rid, id) });
+  }
+
+  experiments.sort((a, b) => b.exp.createdAt - a.exp.createdAt);
+
+  let count = 0;
+  for (const { id, exp } of experiments) {
+    const reproCount = confirmedCount(exp);
+
+    if (values.reproduced && reproCount === 0) continue;
+    if (values.unverified && reproCount > 0) continue;
+
+    count++;
+    const reproLabel = reproCount > 0 ? ` [${reproCount} verified]` : "";
+
+    console.log(
+      `${shortId(id)} ${exp.metricName} ${deltaDisplay(exp.deltaPctX100)}${reproLabel}`,
+    );
+  }
+
+  if (count === 0) {
+    console.log("No experiments found.");
+  }
+}

package/src/cli/commands/publish.ts

@@ -0,0 +1,68 @@
+import { parseArgs } from "node:util";
+import { buildPublishAction } from "../../cob/actions.js";
+import { MEASUREMENT_OPTIONS, buildMeasurement, parseSecondary, requireArg, requireInt } from "../helpers.js";
+import { deltaDisplay, measurementDisplay } from "../format.js";
+import { getRepoId, cobCreate } from "../rad.js";
+
+export function cmdPublish(args: string[]): void {
+  const { values } = parseArgs({
+    args,
+    options: {
+      repo: { type: "string", short: "r" },
+      description: { type: "string", short: "d" },
+      base: { type: "string" },
+      head: { type: "string" },
+      metric: { type: "string" },
+      unit: { type: "string" },
+      direction: { type: "string" },
+      runner: { type: "string" },
+      ...MEASUREMENT_OPTIONS,
+      secondary: { type: "string", multiple: true, default: [] },
+      "agent-system": { type: "string", default: "claude-code" },
+      "agent-model": { type: "string", default: "claude-opus-4-6" },
+      os: { type: "string", default: "" },
+      cpu: { type: "string", default: "" },
+    },
+    strict: true,
+  });
+
+  const rid = getRepoId(values.repo as string | undefined);
+  const base = requireArg(values, "base");
+  const head = requireArg(values, "head");
+  const metric = requireArg(values, "metric");
+  const unit = requireArg(values, "unit");
+  const direction = requireArg(values, "direction");
+  const runner = requireArg(values, "runner");
+  const delta = requireInt(values, "delta");
+  const baseline = buildMeasurement(values, "baseline");
+  const candidate = buildMeasurement(values, "candidate");
+  const secondaryMetrics = ((values.secondary ?? []) as string[]).map(parseSecondary);
+
+  const action = buildPublishAction({
+    description: (values.description as string) ?? undefined,
+    base,
+    oid: head,
+    metricName: metric,
+    metricUnit: unit,
+    direction,
+    runnerClass: runner,
+    os: (values.os as string) ?? "",
+    cpu: (values.cpu as string) ?? "",
+    baseline,
+    candidate,
+    deltaPctX100: delta,
+    buildOk: true,
+    testsOk: true,
+    sanitizersOk: false,
+    agentSystem: (values["agent-system"] as string) ?? "claude-code",
+    agentModel: (values["agent-model"] as string) ?? "claude-opus-4-6",
+    secondaryMetrics,
+  });
+
+  const objectId = cobCreate(rid, [action], "Publish experiment");
+
+  console.log(`Experiment published: ${objectId}`);
+  console.log(`  metric:    ${metric} ${deltaDisplay(delta)}`);
+  console.log(`  baseline:  ${measurementDisplay(baseline, unit)}`);
+  console.log(`  candidate: ${measurementDisplay(candidate, unit)}`);
+}

package/src/cli/commands/reproduce.ts

@@ -0,0 +1,52 @@
+import { parseArgs } from "node:util";
+import { buildReproduceAction } from "../../cob/actions.js";
+import { MEASUREMENT_OPTIONS, VERDICTS, buildMeasurement, die, requireArg, requireInt } from "../helpers.js";
+import { shortId } from "../format.js";
+import { getRepoId, cobUpdate } from "../rad.js";
+import type { Verdict } from "../../types.js";
+
+export function cmdReproduce(args: string[]): void {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      repo: { type: "string", short: "r" },
+      verdict: { type: "string" },
+      runner: { type: "string" },
+      ...MEASUREMENT_OPTIONS,
+      notes: { type: "string" },
+    },
+    allowPositionals: true,
+    strict: true,
+  });
+
+  if (positionals.length === 0) die("missing experiment ID");
+  const id = positionals[0];
+  const rid = getRepoId(values.repo as string | undefined);
+
+  const verdictStr = requireArg(values, "verdict").toLowerCase();
+  if (!VERDICTS.includes(verdictStr as Verdict)) {
+    die(`Invalid verdict: unknown verdict: ${verdictStr}`);
+  }
+  const verdict = verdictStr as Verdict;
+
+  const runner = requireArg(values, "runner");
+  const delta = requireInt(values, "delta");
+  const baseline = buildMeasurement(values, "baseline");
+  const candidate = buildMeasurement(values, "candidate");
+
+  const action = buildReproduceAction({
+    verdict,
+    runnerClass: runner,
+    baseline,
+    candidate,
+    deltaPctX100: delta,
+    buildOk: true,
+    testsOk: true,
+    notes: (values.notes as string) ?? undefined,
+  });
+
+  cobUpdate(rid, id, [action], "Reproduce");
+
+  console.log(`Reproduction added to ${shortId(id)}`);
+  console.log(`  verdict: ${verdict}`);
+}

package/src/cli/commands/show.ts

@@ -0,0 +1,70 @@
+import { parseArgs } from "node:util";
+import { die } from "../helpers.js";
+import { deltaDisplay, measurementDisplay } from "../format.js";
+import { getRepoId, cobShow } from "../rad.js";
+import type { Experiment } from "../../types.js";
+
+export function cmdShow(args: string[]): void {
+  const { values, positionals } = parseArgs({
+    args,
+    options: {
+      repo: { type: "string", short: "r" },
+      json: { type: "boolean", default: false },
+    },
+    allowPositionals: true,
+    strict: true,
+  });
+
+  if (positionals.length === 0) die("missing experiment ID");
+  const id = positionals[0];
+  const rid = getRepoId(values.repo as string | undefined);
+
+  let exp: Experiment;
+  try {
+    exp = cobShow(rid, id);
+  } catch {
+    die(`Experiment not found: ${id}`);
+  }
+
+  if (values.json) {
+    console.log(JSON.stringify(exp, null, 2));
+  } else {
+    console.log(`Experiment ${id}`);
+    if (exp.description) {
+      console.log();
+      console.log(`  ${exp.description}`);
+    }
+    console.log();
+    console.log(`  base:         ${exp.base}`);
+    console.log(`  head:         ${exp.oid}`);
+    console.log();
+    console.log(`  metric:       ${exp.metricName} (${exp.metricUnit})`);
+    console.log(`  direction:    ${exp.direction}`);
+    console.log();
+    console.log(
+      `  baseline:     ${measurementDisplay(exp.baseline, exp.metricUnit)} (n=${exp.baseline.n})`,
+    );
+    console.log(
+      `  candidate:    ${measurementDisplay(exp.candidate, exp.metricUnit)} (n=${exp.candidate.n})`,
+    );
+    console.log(`  delta:        ${deltaDisplay(exp.deltaPctX100)}`);
+    console.log();
+    console.log(`  runner:       ${exp.runnerClass} (${exp.os}, ${exp.cpu})`);
+    console.log(`  build:        ${exp.buildOk ? "ok" : "FAIL"}`);
+    console.log(`  tests:        ${exp.testsOk ? "ok" : "FAIL"}`);
+    console.log(`  agent:        ${exp.agentSystem}/${exp.agentModel}`);
+    console.log(`  author:       ${exp.author.id}`);
+
+    if (exp.reproductions && exp.reproductions.length > 0) {
+      console.log();
+      console.log(`  Reproductions (${exp.reproductions.length}):`);
+      for (const r of exp.reproductions) {
+        const sign = r.deltaPctX100 >= 0 ? "+" : "";
+        const abs = Math.abs(r.deltaPctX100 % 100);
+        console.log(
+          `    ${r.verdict} by ${r.author.id} on ${r.runnerClass} (${sign}${Math.trunc(r.deltaPctX100 / 100)}.${String(abs).padStart(2, "0")}%)`,
+        );
+      }
+    }
+  }
+}

package/src/cli/format.ts

@@ -0,0 +1,27 @@
+// Display formatting helpers — mirrors the Rust Display impls.
+
+import type { Experiment, Measurement } from "../types.js";
+
+/** Format delta as "+5.47%" or "-0.78%". */
+export function deltaDisplay(deltaPctX100: number): string {
+  const abs = Math.abs(deltaPctX100);
+  const sign = deltaPctX100 >= 0 ? "+" : "-";
+  return `${sign}${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}%`;
+}
+
+/** Format measurement as "59.340 ms". */
+export function measurementDisplay(m: Measurement, unit: string): string {
+  const whole = Math.floor(m.medianX1000 / 1000);
+  const frac = Math.abs(m.medianX1000 % 1000);
+  return `${whole}.${frac} ${unit}`;
+}
+
+/** First 7 chars of an ID. */
+export function shortId(id: string): string {
+  return id.substring(0, 7);
+}
+
+/** Confirmed reproduction count. */
+export function confirmedCount(exp: Experiment): number {
+  return exp.reproductions.filter((r) => r.verdict === "confirmed").length;
+}