rad-experiment 0.1.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +87 -0
- package/dist/cli/commands/list.d.ts +1 -0
- package/dist/cli/commands/list.js +35 -0
- package/dist/cli/commands/publish.d.ts +1 -0
- package/dist/cli/commands/publish.js +63 -0
- package/dist/cli/commands/reproduce.d.ts +1 -0
- package/dist/cli/commands/reproduce.js +45 -0
- package/dist/cli/commands/show.d.ts +1 -0
- package/dist/cli/commands/show.js +61 -0
- package/dist/cli/format.d.ts +9 -0
- package/dist/cli/format.js +21 -0
- package/dist/cli/helpers.d.ts +49 -0
- package/dist/cli/helpers.js +90 -0
- package/dist/cli/rad.d.ts +11 -0
- package/dist/cli/rad.js +64 -0
- package/dist/cob/actions.d.ts +35 -0
- package/dist/cob/actions.js +57 -0
- package/dist/cob/state.d.ts +7 -0
- package/dist/cob/state.js +97 -0
- package/dist/rad-cob-experiment.d.ts +2 -0
- package/dist/rad-cob-experiment.js +33 -0
- package/dist/rad-experiment.d.ts +2 -0
- package/dist/rad-experiment.js +74 -0
- package/dist/types.d.ts +102 -0
- package/dist/types.js +9 -0
- package/package.json +24 -0
- package/src/__tests__/actions.test.ts +122 -0
- package/src/__tests__/cob-protocol.test.ts +138 -0
- package/src/__tests__/fixtures.ts +119 -0
- package/src/__tests__/format.test.ts +55 -0
- package/src/__tests__/golden/publish-action.json +46 -0
- package/src/__tests__/golden/publish-minimal.json +25 -0
- package/src/__tests__/golden/publish-with-samples.json +38 -0
- package/src/__tests__/golden/reproduce-action.json +19 -0
- package/src/__tests__/golden/reproduce-minimal.json +18 -0
- package/src/__tests__/helpers.test.ts +138 -0
- package/src/__tests__/integration.test.ts +124 -0
- package/src/__tests__/serialization.test.ts +175 -0
- package/src/__tests__/state.test.ts +191 -0
- package/src/cli/commands/list.ts +45 -0
- package/src/cli/commands/publish.ts +68 -0
- package/src/cli/commands/reproduce.ts +52 -0
- package/src/cli/commands/show.ts +70 -0
- package/src/cli/format.ts +27 -0
- package/src/cli/helpers.ts +101 -0
- package/src/cli/rad.ts +87 -0
- package/src/cob/actions.ts +100 -0
- package/src/cob/state.ts +120 -0
- package/src/rad-cob-experiment.ts +39 -0
- package/src/rad-experiment.ts +85 -0
- package/src/types.ts +133 -0
- package/tsconfig.json +16 -0
package/README.md
ADDED
|
@@ -0,0 +1,87 @@
|
|
|
1
|
+
# rad-experiment (TypeScript)
|
|
2
|
+
|
|
3
|
+
A TypeScript implementation of the `rad-experiment` CLI and Radicle COB helper for AI-generated optimization experiments. Drop-in replacement for the Rust `rad-experiment` crate — produces byte-identical COBs and is fully cross-compatible.
|
|
4
|
+
|
|
5
|
+
## Architecture
|
|
6
|
+
|
|
7
|
+
The implementation provides two binaries that plug into Radicle's external COB system:
|
|
8
|
+
|
|
9
|
+
```
|
|
10
|
+
rad-experiment publish ...
|
|
11
|
+
│
|
|
12
|
+
▼
|
|
13
|
+
rad cob create ──stdin──▶ rad-cob-experiment ──stdout──▶ new state
|
|
14
|
+
│ (external COB helper)
|
|
15
|
+
▼
|
|
16
|
+
git refs/cobs/cc.experiment/<oid>
|
|
17
|
+
```
|
|
18
|
+
|
|
19
|
+
### `rad-cob-experiment` — External COB helper
|
|
20
|
+
|
|
21
|
+
Radicle's [external COB protocol](https://radicle.xyz) delegates state evaluation to a helper binary named `rad-cob-{type}`. When any `rad cob` command operates on a `cc.experiment` COB, Radicle spawns `rad-cob-experiment` and communicates via JSON Lines on stdin/stdout:
|
|
22
|
+
|
|
23
|
+
1. Radicle sends `{ value, op, concurrent }` on stdin
|
|
24
|
+
2. Helper applies the operation to the current state
|
|
25
|
+
3. Helper writes the new state as a JSON Line to stdout
|
|
26
|
+
4. Repeat for each operation, then exit 0
|
|
27
|
+
|
|
28
|
+
This is the only binary that Radicle itself invokes. It has no CLI flags.
|
|
29
|
+
|
|
30
|
+
### `rad-experiment` — User-facing CLI
|
|
31
|
+
|
|
32
|
+
Thin wrapper that shells out to `rad cob create/update/list/show`. All COB storage, signing, and replication are handled by Radicle — this CLI just constructs the right action JSON and formats output.
|
|
33
|
+
|
|
34
|
+
**Commands:**
|
|
35
|
+
|
|
36
|
+
| Command | What it does |
|
|
37
|
+
|--------------------|------------------------------------------------------------------------|
|
|
38
|
+
| `publish` | Create a new experiment COB with benchmark results |
|
|
39
|
+
| `list` | List all experiments (optional `--reproduced`/`--unverified` filters) |
|
|
40
|
+
| `show <id>` | Display experiment details (text or `--json`) |
|
|
41
|
+
| `reproduce <id>` | Add an independent verification to an experiment |
|
|
42
|
+
|
|
43
|
+
## Source files
|
|
44
|
+
|
|
45
|
+
```
|
|
46
|
+
src/
|
|
47
|
+
├── rad-experiment.ts CLI entry point — arg routing and dispatch
|
|
48
|
+
├── rad-cob-experiment.ts COB helper entry point — stdin/stdout JSON Lines protocol
|
|
49
|
+
├── types.ts Shared type definitions (Experiment, Action, Op, etc.)
|
|
50
|
+
├── cob/ COB domain logic (used by both entry points)
|
|
51
|
+
│ ├── actions.ts Builds Publish/Reproduce action JSON
|
|
52
|
+
│ └── state.ts Replays actions into Experiment state
|
|
53
|
+
└── cli/ Everything specific to the rad-experiment CLI
|
|
54
|
+
├── helpers.ts Arg parsing helpers (die, requireArg, buildMeasurement, etc.)
|
|
55
|
+
├── format.ts Display formatting (deltaDisplay, measurementDisplay, etc.)
|
|
56
|
+
├── rad.ts Wrappers around rad CLI commands (cobCreate, cobShow, etc.)
|
|
57
|
+
└── commands/
|
|
58
|
+
├── publish.ts rad-experiment publish
|
|
59
|
+
├── list.ts rad-experiment list
|
|
60
|
+
├── show.ts rad-experiment show
|
|
61
|
+
└── reproduce.ts rad-experiment reproduce
|
|
62
|
+
```
|
|
63
|
+
|
|
64
|
+
## Field naming conventions
|
|
65
|
+
|
|
66
|
+
The Rust serde serialization produces mixed naming that this implementation matches exactly:
|
|
67
|
+
|
|
68
|
+
| Layer | Convention | Example |
|
|
69
|
+
|-----------------------------------------|-------------|----------------------------------|
|
|
70
|
+
| **Action fields** (stored in COB) | snake_case | `metric_name` , `delta_pct_x100` |
|
|
71
|
+
| **Measurement fields** (nested struct) | camelCase | `medianX1000`, `stdX1000` |
|
|
72
|
+
| **Experiment state** (helper output) | camelCase | `metricName`, `deltaPctX100` |
|
|
73
|
+
|
|
74
|
+
The helper accepts both conventions when reading actions for backward compatibility.
|
|
75
|
+
|
|
76
|
+
## Build and install
|
|
77
|
+
|
|
78
|
+
```sh
|
|
79
|
+
npm install
|
|
80
|
+
npm run build
|
|
81
|
+
|
|
82
|
+
# Symlink both binaries onto PATH
|
|
83
|
+
ln -s "$(pwd)/dist/rad-experiment.js" ~/.local/bin/rad-experiment
|
|
84
|
+
ln -s "$(pwd)/dist/rad-cob-experiment.js" ~/.local/bin/rad-cob-experiment
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Both binaries must be on `$PATH`. Radicle finds `rad-cob-experiment` by name.
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdList(args: string[]): void;
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { deltaDisplay, shortId, confirmedCount } from "../format.js";
|
|
3
|
+
import { getRepoId, cobList, cobShow } from "../rad.js";
|
|
4
|
+
export function cmdList(args) {
|
|
5
|
+
const { values } = parseArgs({
|
|
6
|
+
args,
|
|
7
|
+
options: {
|
|
8
|
+
repo: { type: "string", short: "r" },
|
|
9
|
+
reproduced: { type: "boolean", default: false },
|
|
10
|
+
unverified: { type: "boolean", default: false },
|
|
11
|
+
},
|
|
12
|
+
strict: true,
|
|
13
|
+
});
|
|
14
|
+
const rid = getRepoId(values.repo);
|
|
15
|
+
const ids = cobList(rid);
|
|
16
|
+
const experiments = [];
|
|
17
|
+
for (const id of ids) {
|
|
18
|
+
experiments.push({ id, exp: cobShow(rid, id) });
|
|
19
|
+
}
|
|
20
|
+
experiments.sort((a, b) => b.exp.createdAt - a.exp.createdAt);
|
|
21
|
+
let count = 0;
|
|
22
|
+
for (const { id, exp } of experiments) {
|
|
23
|
+
const reproCount = confirmedCount(exp);
|
|
24
|
+
if (values.reproduced && reproCount === 0)
|
|
25
|
+
continue;
|
|
26
|
+
if (values.unverified && reproCount > 0)
|
|
27
|
+
continue;
|
|
28
|
+
count++;
|
|
29
|
+
const reproLabel = reproCount > 0 ? ` [${reproCount} verified]` : "";
|
|
30
|
+
console.log(`${shortId(id)} ${exp.metricName} ${deltaDisplay(exp.deltaPctX100)}${reproLabel}`);
|
|
31
|
+
}
|
|
32
|
+
if (count === 0) {
|
|
33
|
+
console.log("No experiments found.");
|
|
34
|
+
}
|
|
35
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdPublish(args: string[]): void;
|
|
@@ -0,0 +1,63 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { buildPublishAction } from "../../cob/actions.js";
|
|
3
|
+
import { MEASUREMENT_OPTIONS, buildMeasurement, parseSecondary, requireArg, requireInt } from "../helpers.js";
|
|
4
|
+
import { deltaDisplay, measurementDisplay } from "../format.js";
|
|
5
|
+
import { getRepoId, cobCreate } from "../rad.js";
|
|
6
|
+
export function cmdPublish(args) {
|
|
7
|
+
const { values } = parseArgs({
|
|
8
|
+
args,
|
|
9
|
+
options: {
|
|
10
|
+
repo: { type: "string", short: "r" },
|
|
11
|
+
description: { type: "string", short: "d" },
|
|
12
|
+
base: { type: "string" },
|
|
13
|
+
head: { type: "string" },
|
|
14
|
+
metric: { type: "string" },
|
|
15
|
+
unit: { type: "string" },
|
|
16
|
+
direction: { type: "string" },
|
|
17
|
+
runner: { type: "string" },
|
|
18
|
+
...MEASUREMENT_OPTIONS,
|
|
19
|
+
secondary: { type: "string", multiple: true, default: [] },
|
|
20
|
+
"agent-system": { type: "string", default: "claude-code" },
|
|
21
|
+
"agent-model": { type: "string", default: "claude-opus-4-6" },
|
|
22
|
+
os: { type: "string", default: "" },
|
|
23
|
+
cpu: { type: "string", default: "" },
|
|
24
|
+
},
|
|
25
|
+
strict: true,
|
|
26
|
+
});
|
|
27
|
+
const rid = getRepoId(values.repo);
|
|
28
|
+
const base = requireArg(values, "base");
|
|
29
|
+
const head = requireArg(values, "head");
|
|
30
|
+
const metric = requireArg(values, "metric");
|
|
31
|
+
const unit = requireArg(values, "unit");
|
|
32
|
+
const direction = requireArg(values, "direction");
|
|
33
|
+
const runner = requireArg(values, "runner");
|
|
34
|
+
const delta = requireInt(values, "delta");
|
|
35
|
+
const baseline = buildMeasurement(values, "baseline");
|
|
36
|
+
const candidate = buildMeasurement(values, "candidate");
|
|
37
|
+
const secondaryMetrics = (values.secondary ?? []).map(parseSecondary);
|
|
38
|
+
const action = buildPublishAction({
|
|
39
|
+
description: values.description ?? undefined,
|
|
40
|
+
base,
|
|
41
|
+
oid: head,
|
|
42
|
+
metricName: metric,
|
|
43
|
+
metricUnit: unit,
|
|
44
|
+
direction,
|
|
45
|
+
runnerClass: runner,
|
|
46
|
+
os: values.os ?? "",
|
|
47
|
+
cpu: values.cpu ?? "",
|
|
48
|
+
baseline,
|
|
49
|
+
candidate,
|
|
50
|
+
deltaPctX100: delta,
|
|
51
|
+
buildOk: true,
|
|
52
|
+
testsOk: true,
|
|
53
|
+
sanitizersOk: false,
|
|
54
|
+
agentSystem: values["agent-system"] ?? "claude-code",
|
|
55
|
+
agentModel: values["agent-model"] ?? "claude-opus-4-6",
|
|
56
|
+
secondaryMetrics,
|
|
57
|
+
});
|
|
58
|
+
const objectId = cobCreate(rid, [action], "Publish experiment");
|
|
59
|
+
console.log(`Experiment published: ${objectId}`);
|
|
60
|
+
console.log(` metric: ${metric} ${deltaDisplay(delta)}`);
|
|
61
|
+
console.log(` baseline: ${measurementDisplay(baseline, unit)}`);
|
|
62
|
+
console.log(` candidate: ${measurementDisplay(candidate, unit)}`);
|
|
63
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdReproduce(args: string[]): void;
|
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { buildReproduceAction } from "../../cob/actions.js";
|
|
3
|
+
import { MEASUREMENT_OPTIONS, VERDICTS, buildMeasurement, die, requireArg, requireInt } from "../helpers.js";
|
|
4
|
+
import { shortId } from "../format.js";
|
|
5
|
+
import { getRepoId, cobUpdate } from "../rad.js";
|
|
6
|
+
export function cmdReproduce(args) {
|
|
7
|
+
const { values, positionals } = parseArgs({
|
|
8
|
+
args,
|
|
9
|
+
options: {
|
|
10
|
+
repo: { type: "string", short: "r" },
|
|
11
|
+
verdict: { type: "string" },
|
|
12
|
+
runner: { type: "string" },
|
|
13
|
+
...MEASUREMENT_OPTIONS,
|
|
14
|
+
notes: { type: "string" },
|
|
15
|
+
},
|
|
16
|
+
allowPositionals: true,
|
|
17
|
+
strict: true,
|
|
18
|
+
});
|
|
19
|
+
if (positionals.length === 0)
|
|
20
|
+
die("missing experiment ID");
|
|
21
|
+
const id = positionals[0];
|
|
22
|
+
const rid = getRepoId(values.repo);
|
|
23
|
+
const verdictStr = requireArg(values, "verdict").toLowerCase();
|
|
24
|
+
if (!VERDICTS.includes(verdictStr)) {
|
|
25
|
+
die(`Invalid verdict: unknown verdict: ${verdictStr}`);
|
|
26
|
+
}
|
|
27
|
+
const verdict = verdictStr;
|
|
28
|
+
const runner = requireArg(values, "runner");
|
|
29
|
+
const delta = requireInt(values, "delta");
|
|
30
|
+
const baseline = buildMeasurement(values, "baseline");
|
|
31
|
+
const candidate = buildMeasurement(values, "candidate");
|
|
32
|
+
const action = buildReproduceAction({
|
|
33
|
+
verdict,
|
|
34
|
+
runnerClass: runner,
|
|
35
|
+
baseline,
|
|
36
|
+
candidate,
|
|
37
|
+
deltaPctX100: delta,
|
|
38
|
+
buildOk: true,
|
|
39
|
+
testsOk: true,
|
|
40
|
+
notes: values.notes ?? undefined,
|
|
41
|
+
});
|
|
42
|
+
cobUpdate(rid, id, [action], "Reproduce");
|
|
43
|
+
console.log(`Reproduction added to ${shortId(id)}`);
|
|
44
|
+
console.log(` verdict: ${verdict}`);
|
|
45
|
+
}
|
|
@@ -0,0 +1 @@
|
|
|
1
|
+
export declare function cmdShow(args: string[]): void;
|
|
@@ -0,0 +1,61 @@
|
|
|
1
|
+
import { parseArgs } from "node:util";
|
|
2
|
+
import { die } from "../helpers.js";
|
|
3
|
+
import { deltaDisplay, measurementDisplay } from "../format.js";
|
|
4
|
+
import { getRepoId, cobShow } from "../rad.js";
|
|
5
|
+
export function cmdShow(args) {
|
|
6
|
+
const { values, positionals } = parseArgs({
|
|
7
|
+
args,
|
|
8
|
+
options: {
|
|
9
|
+
repo: { type: "string", short: "r" },
|
|
10
|
+
json: { type: "boolean", default: false },
|
|
11
|
+
},
|
|
12
|
+
allowPositionals: true,
|
|
13
|
+
strict: true,
|
|
14
|
+
});
|
|
15
|
+
if (positionals.length === 0)
|
|
16
|
+
die("missing experiment ID");
|
|
17
|
+
const id = positionals[0];
|
|
18
|
+
const rid = getRepoId(values.repo);
|
|
19
|
+
let exp;
|
|
20
|
+
try {
|
|
21
|
+
exp = cobShow(rid, id);
|
|
22
|
+
}
|
|
23
|
+
catch {
|
|
24
|
+
die(`Experiment not found: ${id}`);
|
|
25
|
+
}
|
|
26
|
+
if (values.json) {
|
|
27
|
+
console.log(JSON.stringify(exp, null, 2));
|
|
28
|
+
}
|
|
29
|
+
else {
|
|
30
|
+
console.log(`Experiment ${id}`);
|
|
31
|
+
if (exp.description) {
|
|
32
|
+
console.log();
|
|
33
|
+
console.log(` ${exp.description}`);
|
|
34
|
+
}
|
|
35
|
+
console.log();
|
|
36
|
+
console.log(` base: ${exp.base}`);
|
|
37
|
+
console.log(` head: ${exp.oid}`);
|
|
38
|
+
console.log();
|
|
39
|
+
console.log(` metric: ${exp.metricName} (${exp.metricUnit})`);
|
|
40
|
+
console.log(` direction: ${exp.direction}`);
|
|
41
|
+
console.log();
|
|
42
|
+
console.log(` baseline: ${measurementDisplay(exp.baseline, exp.metricUnit)} (n=${exp.baseline.n})`);
|
|
43
|
+
console.log(` candidate: ${measurementDisplay(exp.candidate, exp.metricUnit)} (n=${exp.candidate.n})`);
|
|
44
|
+
console.log(` delta: ${deltaDisplay(exp.deltaPctX100)}`);
|
|
45
|
+
console.log();
|
|
46
|
+
console.log(` runner: ${exp.runnerClass} (${exp.os}, ${exp.cpu})`);
|
|
47
|
+
console.log(` build: ${exp.buildOk ? "ok" : "FAIL"}`);
|
|
48
|
+
console.log(` tests: ${exp.testsOk ? "ok" : "FAIL"}`);
|
|
49
|
+
console.log(` agent: ${exp.agentSystem}/${exp.agentModel}`);
|
|
50
|
+
console.log(` author: ${exp.author.id}`);
|
|
51
|
+
if (exp.reproductions && exp.reproductions.length > 0) {
|
|
52
|
+
console.log();
|
|
53
|
+
console.log(` Reproductions (${exp.reproductions.length}):`);
|
|
54
|
+
for (const r of exp.reproductions) {
|
|
55
|
+
const sign = r.deltaPctX100 >= 0 ? "+" : "";
|
|
56
|
+
const abs = Math.abs(r.deltaPctX100 % 100);
|
|
57
|
+
console.log(` ${r.verdict} by ${r.author.id} on ${r.runnerClass} (${sign}${Math.trunc(r.deltaPctX100 / 100)}.${String(abs).padStart(2, "0")}%)`);
|
|
58
|
+
}
|
|
59
|
+
}
|
|
60
|
+
}
|
|
61
|
+
}
|
|
@@ -0,0 +1,9 @@
|
|
|
1
|
+
import type { Experiment, Measurement } from "../types.js";
|
|
2
|
+
/** Format delta as "+5.47%" or "-0.78%". */
|
|
3
|
+
export declare function deltaDisplay(deltaPctX100: number): string;
|
|
4
|
+
/** Format measurement as "59.340 ms". */
|
|
5
|
+
export declare function measurementDisplay(m: Measurement, unit: string): string;
|
|
6
|
+
/** First 7 chars of an ID. */
|
|
7
|
+
export declare function shortId(id: string): string;
|
|
8
|
+
/** Confirmed reproduction count. */
|
|
9
|
+
export declare function confirmedCount(exp: Experiment): number;
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
// Display formatting helpers — mirrors the Rust Display impls.
|
|
2
|
+
/** Format delta as "+5.47%" or "-0.78%". */
|
|
3
|
+
export function deltaDisplay(deltaPctX100) {
|
|
4
|
+
const abs = Math.abs(deltaPctX100);
|
|
5
|
+
const sign = deltaPctX100 >= 0 ? "+" : "-";
|
|
6
|
+
return `${sign}${Math.floor(abs / 100)}.${String(abs % 100).padStart(2, "0")}%`;
|
|
7
|
+
}
|
|
8
|
+
/** Format measurement as "59.340 ms". */
|
|
9
|
+
export function measurementDisplay(m, unit) {
|
|
10
|
+
const whole = Math.floor(m.medianX1000 / 1000);
|
|
11
|
+
const frac = Math.abs(m.medianX1000 % 1000);
|
|
12
|
+
return `${whole}.${frac} ${unit}`;
|
|
13
|
+
}
|
|
14
|
+
/** First 7 chars of an ID. */
|
|
15
|
+
export function shortId(id) {
|
|
16
|
+
return id.substring(0, 7);
|
|
17
|
+
}
|
|
18
|
+
/** Confirmed reproduction count. */
|
|
19
|
+
export function confirmedCount(exp) {
|
|
20
|
+
return exp.reproductions.filter((r) => r.verdict === "confirmed").length;
|
|
21
|
+
}
|
|
@@ -0,0 +1,49 @@
|
|
|
1
|
+
import type { Measurement, MetricValue, Verdict } from "../types.js";
|
|
2
|
+
export declare const VERDICTS: Verdict[];
|
|
3
|
+
export declare const MEASUREMENT_OPTIONS: {
|
|
4
|
+
"baseline-median": {
|
|
5
|
+
type: "string";
|
|
6
|
+
};
|
|
7
|
+
"baseline-std": {
|
|
8
|
+
type: "string";
|
|
9
|
+
default: string;
|
|
10
|
+
};
|
|
11
|
+
"baseline-samples": {
|
|
12
|
+
type: "string";
|
|
13
|
+
default: string;
|
|
14
|
+
};
|
|
15
|
+
"baseline-n": {
|
|
16
|
+
type: "string";
|
|
17
|
+
};
|
|
18
|
+
"candidate-median": {
|
|
19
|
+
type: "string";
|
|
20
|
+
};
|
|
21
|
+
"candidate-std": {
|
|
22
|
+
type: "string";
|
|
23
|
+
default: string;
|
|
24
|
+
};
|
|
25
|
+
"candidate-samples": {
|
|
26
|
+
type: "string";
|
|
27
|
+
default: string;
|
|
28
|
+
};
|
|
29
|
+
"candidate-n": {
|
|
30
|
+
type: "string";
|
|
31
|
+
};
|
|
32
|
+
delta: {
|
|
33
|
+
type: "string";
|
|
34
|
+
};
|
|
35
|
+
};
|
|
36
|
+
/** Error thrown by die() — caught at the CLI entry point. */
|
|
37
|
+
export declare class CliError extends Error {
|
|
38
|
+
constructor(message: string);
|
|
39
|
+
}
|
|
40
|
+
export declare function die(msg: string): never;
|
|
41
|
+
export declare function requireArg(values: Record<string, unknown>, name: string): string;
|
|
42
|
+
export declare function requireInt(values: Record<string, unknown>, name: string): number;
|
|
43
|
+
export declare function optionalInt(values: Record<string, unknown>, name: string, defaultVal: number): number;
|
|
44
|
+
export declare function buildMeasurement(values: Record<string, unknown>, prefix: "baseline" | "candidate"): Measurement;
|
|
45
|
+
/**
|
|
46
|
+
* Parse "name:unit:baseline_x1000:candidate_x1000:delta_x100[:regressed]"
|
|
47
|
+
* Mirrors parse_secondary in the Rust CLI.
|
|
48
|
+
*/
|
|
49
|
+
export declare function parseSecondary(s: string): MetricValue;
|
|
@@ -0,0 +1,90 @@
|
|
|
1
|
+
// Shared CLI helpers for arg parsing and measurement construction.
|
|
2
|
+
export const VERDICTS = ["confirmed", "failed", "inconclusive"];
|
|
3
|
+
// Shared parseArgs option specs for baseline/candidate measurement fields.
|
|
4
|
+
export const MEASUREMENT_OPTIONS = {
|
|
5
|
+
"baseline-median": { type: "string" },
|
|
6
|
+
"baseline-std": { type: "string", default: "0" },
|
|
7
|
+
"baseline-samples": { type: "string", default: "" },
|
|
8
|
+
"baseline-n": { type: "string" },
|
|
9
|
+
"candidate-median": { type: "string" },
|
|
10
|
+
"candidate-std": { type: "string", default: "0" },
|
|
11
|
+
"candidate-samples": { type: "string", default: "" },
|
|
12
|
+
"candidate-n": { type: "string" },
|
|
13
|
+
delta: { type: "string" },
|
|
14
|
+
};
|
|
15
|
+
/** Error thrown by die() — caught at the CLI entry point. */
|
|
16
|
+
export class CliError extends Error {
|
|
17
|
+
constructor(message) {
|
|
18
|
+
super(message);
|
|
19
|
+
this.name = "CliError";
|
|
20
|
+
}
|
|
21
|
+
}
|
|
22
|
+
export function die(msg) {
|
|
23
|
+
throw new CliError(msg);
|
|
24
|
+
}
|
|
25
|
+
export function requireArg(values, name) {
|
|
26
|
+
const val = values[name];
|
|
27
|
+
if (val == null || val === "") {
|
|
28
|
+
die(`missing required argument: --${name}`);
|
|
29
|
+
}
|
|
30
|
+
return String(val);
|
|
31
|
+
}
|
|
32
|
+
export function requireInt(values, name) {
|
|
33
|
+
const val = requireArg(values, name);
|
|
34
|
+
const n = parseInt(val, 10);
|
|
35
|
+
if (isNaN(n))
|
|
36
|
+
die(`invalid integer for --${name}: ${val}`);
|
|
37
|
+
return n;
|
|
38
|
+
}
|
|
39
|
+
export function optionalInt(values, name, defaultVal) {
|
|
40
|
+
const val = values[name];
|
|
41
|
+
if (val == null || val === "")
|
|
42
|
+
return defaultVal;
|
|
43
|
+
const n = parseInt(String(val), 10);
|
|
44
|
+
if (isNaN(n))
|
|
45
|
+
die(`invalid integer for --${name}: ${val}`);
|
|
46
|
+
return n;
|
|
47
|
+
}
|
|
48
|
+
function parseSamples(s) {
|
|
49
|
+
if (!s || s.trim() === "")
|
|
50
|
+
return [];
|
|
51
|
+
return s
|
|
52
|
+
.split(",")
|
|
53
|
+
.map((v) => parseInt(v.trim(), 10))
|
|
54
|
+
.filter((v) => !isNaN(v));
|
|
55
|
+
}
|
|
56
|
+
export function buildMeasurement(values, prefix) {
|
|
57
|
+
const median = requireInt(values, `${prefix}-median`);
|
|
58
|
+
const std = optionalInt(values, `${prefix}-std`, 0);
|
|
59
|
+
const n = requireInt(values, `${prefix}-n`);
|
|
60
|
+
const samples = parseSamples(values[`${prefix}-samples`] ?? "");
|
|
61
|
+
const m = { n, medianX1000: median, stdX1000: std };
|
|
62
|
+
if (samples.length > 0)
|
|
63
|
+
m.samplesX1000 = samples;
|
|
64
|
+
return m;
|
|
65
|
+
}
|
|
66
|
+
/**
|
|
67
|
+
* Parse "name:unit:baseline_x1000:candidate_x1000:delta_x100[:regressed]"
|
|
68
|
+
* Mirrors parse_secondary in the Rust CLI.
|
|
69
|
+
*/
|
|
70
|
+
export function parseSecondary(s) {
|
|
71
|
+
const parts = s.split(":");
|
|
72
|
+
if (parts.length < 5) {
|
|
73
|
+
die(`secondary metric needs at least 5 colon-separated fields: ${s}`);
|
|
74
|
+
}
|
|
75
|
+
const baselineMedian = parseInt(parts[2], 10);
|
|
76
|
+
const candidateMedian = parseInt(parts[3], 10);
|
|
77
|
+
const delta = parseInt(parts[4], 10);
|
|
78
|
+
if (isNaN(baselineMedian) || isNaN(candidateMedian) || isNaN(delta)) {
|
|
79
|
+
die(`secondary metric has non-integer numeric fields: ${s}`);
|
|
80
|
+
}
|
|
81
|
+
const regressed = parts[5] === "true";
|
|
82
|
+
return {
|
|
83
|
+
name: parts[0],
|
|
84
|
+
unit: parts[1],
|
|
85
|
+
baseline: { n: 1, medianX1000: baselineMedian, stdX1000: 0 },
|
|
86
|
+
candidate: { n: 1, medianX1000: candidateMedian, stdX1000: 0 },
|
|
87
|
+
deltaPctX100: delta,
|
|
88
|
+
regressed,
|
|
89
|
+
};
|
|
90
|
+
}
|
|
@@ -0,0 +1,11 @@
|
|
|
1
|
+
import type { Action, Experiment } from "../types.js";
|
|
2
|
+
/** Resolve the RID for a repository path. */
|
|
3
|
+
export declare function getRepoId(repoPath?: string): string;
|
|
4
|
+
/** Create a new COB. Returns the ObjectId. */
|
|
5
|
+
export declare function cobCreate(rid: string, actions: Action[], message: string): string;
|
|
6
|
+
/** Update an existing COB. */
|
|
7
|
+
export declare function cobUpdate(rid: string, objectId: string, actions: Action[], message: string): string;
|
|
8
|
+
/** List all COB object IDs for cc.experiment type. */
|
|
9
|
+
export declare function cobList(rid: string): string[];
|
|
10
|
+
/** Show a COB's state as JSON. */
|
|
11
|
+
export declare function cobShow(rid: string, objectId: string): Experiment;
|
package/dist/cli/rad.js
ADDED
|
@@ -0,0 +1,64 @@
|
|
|
1
|
+
// Wrapper around `rad` CLI commands for COB operations.
|
|
2
|
+
import { execFileSync } from "node:child_process";
|
|
3
|
+
import { writeFileSync, rmSync, mkdtempSync } from "node:fs";
|
|
4
|
+
import { join, dirname } from "node:path";
|
|
5
|
+
import { tmpdir } from "node:os";
|
|
6
|
+
const COB_TYPE = "cc.experiment";
|
|
7
|
+
/** Resolve the RID for a repository path. */
|
|
8
|
+
export function getRepoId(repoPath) {
|
|
9
|
+
const args = ["inspect", "--rid"];
|
|
10
|
+
const opts = repoPath ? { cwd: repoPath } : {};
|
|
11
|
+
return execFileSync("rad", args, { encoding: "utf-8", ...opts }).trim();
|
|
12
|
+
}
|
|
13
|
+
/** Write actions to a temp JSONL file, return its path. */
|
|
14
|
+
function writeTempActions(actions) {
|
|
15
|
+
const dir = mkdtempSync(join(tmpdir(), "rad-exp-"));
|
|
16
|
+
const path = join(dir, "actions.jsonl");
|
|
17
|
+
const content = actions.map((a) => JSON.stringify(a)).join("\n") + "\n";
|
|
18
|
+
writeFileSync(path, content);
|
|
19
|
+
return path;
|
|
20
|
+
}
|
|
21
|
+
/** Clean up a temp file and its parent directory (best-effort). */
|
|
22
|
+
function cleanupTemp(path) {
|
|
23
|
+
try {
|
|
24
|
+
rmSync(dirname(path), { recursive: true });
|
|
25
|
+
}
|
|
26
|
+
catch {
|
|
27
|
+
// ignore
|
|
28
|
+
}
|
|
29
|
+
}
|
|
30
|
+
/** Shared implementation for cobCreate and cobUpdate. */
|
|
31
|
+
function cobExec(rid, subcommand, actions, message, objectId) {
|
|
32
|
+
const tmpFile = writeTempActions(actions);
|
|
33
|
+
try {
|
|
34
|
+
const args = ["cob", subcommand, "--repo", rid, "--type", COB_TYPE];
|
|
35
|
+
if (objectId)
|
|
36
|
+
args.push("--object", objectId);
|
|
37
|
+
args.push("--message", message, tmpFile);
|
|
38
|
+
return execFileSync("rad", args, { encoding: "utf-8" }).trim();
|
|
39
|
+
}
|
|
40
|
+
finally {
|
|
41
|
+
cleanupTemp(tmpFile);
|
|
42
|
+
}
|
|
43
|
+
}
|
|
44
|
+
/** Create a new COB. Returns the ObjectId. */
|
|
45
|
+
export function cobCreate(rid, actions, message) {
|
|
46
|
+
return cobExec(rid, "create", actions, message);
|
|
47
|
+
}
|
|
48
|
+
/** Update an existing COB. */
|
|
49
|
+
export function cobUpdate(rid, objectId, actions, message) {
|
|
50
|
+
return cobExec(rid, "update", actions, message, objectId);
|
|
51
|
+
}
|
|
52
|
+
/** List all COB object IDs for cc.experiment type. */
|
|
53
|
+
export function cobList(rid) {
|
|
54
|
+
const stdout = execFileSync("rad", ["cob", "list", "--repo", rid, "--type", COB_TYPE], { encoding: "utf-8" });
|
|
55
|
+
return stdout
|
|
56
|
+
.split("\n")
|
|
57
|
+
.map((s) => s.trim())
|
|
58
|
+
.filter((s) => s.length > 0);
|
|
59
|
+
}
|
|
60
|
+
/** Show a COB's state as JSON. */
|
|
61
|
+
export function cobShow(rid, objectId) {
|
|
62
|
+
const stdout = execFileSync("rad", ["cob", "show", "--repo", rid, "--type", COB_TYPE, "--object", objectId, "--format", "json"], { encoding: "utf-8" });
|
|
63
|
+
return JSON.parse(stdout);
|
|
64
|
+
}
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
import type { Measurement, MetricValue, PublishAction, ReproduceAction, Verdict } from "../types.js";
|
|
2
|
+
/** Strip empty samplesX1000 from a Measurement before serialization. */
|
|
3
|
+
export declare function cleanMeasurement(m: Measurement): Measurement;
|
|
4
|
+
/** Build a Publish action matching exact Rust serde output. */
|
|
5
|
+
export declare function buildPublishAction(opts: {
|
|
6
|
+
description?: string;
|
|
7
|
+
base: string;
|
|
8
|
+
oid: string;
|
|
9
|
+
metricName: string;
|
|
10
|
+
metricUnit: string;
|
|
11
|
+
direction: string;
|
|
12
|
+
runnerClass: string;
|
|
13
|
+
os: string;
|
|
14
|
+
cpu: string;
|
|
15
|
+
baseline: Measurement;
|
|
16
|
+
candidate: Measurement;
|
|
17
|
+
deltaPctX100: number;
|
|
18
|
+
buildOk: boolean;
|
|
19
|
+
testsOk: boolean;
|
|
20
|
+
sanitizersOk: boolean;
|
|
21
|
+
agentSystem: string;
|
|
22
|
+
agentModel: string;
|
|
23
|
+
secondaryMetrics: MetricValue[];
|
|
24
|
+
}): PublishAction;
|
|
25
|
+
/** Build a Reproduce action matching exact Rust serde output. */
|
|
26
|
+
export declare function buildReproduceAction(opts: {
|
|
27
|
+
verdict: Verdict;
|
|
28
|
+
runnerClass: string;
|
|
29
|
+
baseline: Measurement;
|
|
30
|
+
candidate: Measurement;
|
|
31
|
+
deltaPctX100: number;
|
|
32
|
+
buildOk: boolean;
|
|
33
|
+
testsOk: boolean;
|
|
34
|
+
notes?: string;
|
|
35
|
+
}): ReproduceAction;
|
|
@@ -0,0 +1,57 @@
|
|
|
1
|
+
// Action builders — constructs action objects with correct field naming.
|
|
2
|
+
//
|
|
3
|
+
// Actions use snake_case for Action-level fields (matching Rust serde output),
|
|
4
|
+
// but camelCase for nested Measurement/MetricValue fields.
|
|
5
|
+
// Optional/empty fields are omitted (matching Rust skip_serializing_if).
|
|
6
|
+
/** Strip empty samplesX1000 from a Measurement before serialization. */
|
|
7
|
+
export function cleanMeasurement(m) {
|
|
8
|
+
const result = { n: m.n, medianX1000: m.medianX1000, stdX1000: m.stdX1000 };
|
|
9
|
+
if (m.samplesX1000 && m.samplesX1000.length > 0) {
|
|
10
|
+
result.samplesX1000 = m.samplesX1000;
|
|
11
|
+
}
|
|
12
|
+
return result;
|
|
13
|
+
}
|
|
14
|
+
/** Build a Publish action matching exact Rust serde output. */
|
|
15
|
+
export function buildPublishAction(opts) {
|
|
16
|
+
// Field order matches Rust struct declaration for consistent JSON output.
|
|
17
|
+
const action = { type: "publish" };
|
|
18
|
+
if (opts.description != null)
|
|
19
|
+
action.description = opts.description;
|
|
20
|
+
// Git object dependencies — prevents GC from pruning commits.
|
|
21
|
+
action.parents = [opts.base, opts.oid];
|
|
22
|
+
action.base = opts.base;
|
|
23
|
+
action.oid = opts.oid;
|
|
24
|
+
action.metric_name = opts.metricName;
|
|
25
|
+
action.metric_unit = opts.metricUnit;
|
|
26
|
+
action.direction = opts.direction;
|
|
27
|
+
action.runner_class = opts.runnerClass;
|
|
28
|
+
if (opts.os !== "")
|
|
29
|
+
action.os = opts.os;
|
|
30
|
+
if (opts.cpu !== "")
|
|
31
|
+
action.cpu = opts.cpu;
|
|
32
|
+
action.baseline = cleanMeasurement(opts.baseline);
|
|
33
|
+
action.candidate = cleanMeasurement(opts.candidate);
|
|
34
|
+
action.delta_pct_x100 = Math.abs(opts.deltaPctX100);
|
|
35
|
+
action.build_ok = opts.buildOk;
|
|
36
|
+
action.tests_ok = opts.testsOk;
|
|
37
|
+
action.sanitizers_ok = opts.sanitizersOk;
|
|
38
|
+
action.agent_system = opts.agentSystem;
|
|
39
|
+
action.agent_model = opts.agentModel;
|
|
40
|
+
if (opts.secondaryMetrics.length > 0)
|
|
41
|
+
action.secondary_metrics = opts.secondaryMetrics;
|
|
42
|
+
return action;
|
|
43
|
+
}
|
|
44
|
+
/** Build a Reproduce action matching exact Rust serde output. */
|
|
45
|
+
export function buildReproduceAction(opts) {
|
|
46
|
+
const action = { type: "reproduce" };
|
|
47
|
+
action.verdict = opts.verdict;
|
|
48
|
+
action.runner_class = opts.runnerClass;
|
|
49
|
+
action.baseline = cleanMeasurement(opts.baseline);
|
|
50
|
+
action.candidate = cleanMeasurement(opts.candidate);
|
|
51
|
+
action.delta_pct_x100 = Math.abs(opts.deltaPctX100);
|
|
52
|
+
action.build_ok = opts.buildOk;
|
|
53
|
+
action.tests_ok = opts.testsOk;
|
|
54
|
+
if (opts.notes != null)
|
|
55
|
+
action.notes = opts.notes;
|
|
56
|
+
return action;
|
|
57
|
+
}
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
import type { Experiment, Op, OpMessage } from "../types.js";
|
|
2
|
+
/** Create an Experiment from the root operation. Mirrors Experiment::from_root. */
|
|
3
|
+
export declare function fromRoot(op: Op): Experiment;
|
|
4
|
+
/** Apply a subsequent operation to an existing experiment. */
|
|
5
|
+
export declare function applyOp(exp: Experiment, op: Op): void;
|
|
6
|
+
/** Process an OpMessage from Radicle's external COB protocol. */
|
|
7
|
+
export declare function handleOpMessage(msg: OpMessage): Experiment;
|