ralphctl 0.17.1 → 0.19.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +113 -53
- package/dist/{cli-WTLOGLWF.mjs → cli-4CEINUTM.mjs} +13991 -11488
- package/dist/cli.mjs +1 -1
- package/dist/manifest.json +4 -2
- package/dist/prompts/evaluate/template.md +15 -5
- package/dist/prompts/evaluate-continuation/template.md +18 -8
- package/dist/prompts/implement/template.md +24 -9
- package/dist/prompts/implement-continuation/template.md +16 -10
- package/dist/prompts/plan/template.md +7 -1
- package/dist/prompts/reproduce/template.md +127 -0
- package/dist/prompts/select-candidate/template.md +99 -0
- package/package.json +14 -13
package/dist/cli.mjs
CHANGED
package/dist/manifest.json
CHANGED
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": 1,
|
|
3
|
-
"packageVersion": "0.
|
|
4
|
-
"generatedAt": "2026-
|
|
3
|
+
"packageVersion": "0.19.0",
|
|
4
|
+
"generatedAt": "2026-08-11T05:49:20.299Z",
|
|
5
5
|
"assets": [
|
|
6
6
|
"agent-definitions/ralphctl-evaluator.md",
|
|
7
7
|
"agent-definitions/ralphctl-generator.md",
|
|
@@ -24,6 +24,8 @@
|
|
|
24
24
|
"prompts/plan/template.md",
|
|
25
25
|
"prompts/readiness/template.md",
|
|
26
26
|
"prompts/refine/template.md",
|
|
27
|
+
"prompts/reproduce/template.md",
|
|
28
|
+
"prompts/select-candidate/template.md",
|
|
27
29
|
"skills/ralphctl-abstraction-first/SKILL.md",
|
|
28
30
|
"skills/ralphctl-alignment/SKILL.md",
|
|
29
31
|
"skills/ralphctl-cherny-workflow/SKILL.md",
|
|
@@ -119,6 +119,8 @@ end. The final `signals.json` is the only machine-readable output and must come
|
|
|
119
119
|
<project_tooling>{{PROJECT_TOOLING}}</project_tooling>
|
|
120
120
|
<prior_progress>{{PRIOR_PROGRESS}}</prior_progress>
|
|
121
121
|
|
|
122
|
+
{{REPRODUCTION_SECTION}}
|
|
123
|
+
|
|
122
124
|
{{GENERATOR_HINTS_SECTION}}
|
|
123
125
|
</inputs>
|
|
124
126
|
|
|
@@ -212,6 +214,9 @@ Run deterministic checks first — they are authoritative and cheap.
|
|
|
212
214
|
4. **Audit the diff for verification tampering** — check whether the changes touch test files, fixtures, or
|
|
213
215
|
verification tooling themselves. A criterion satisfied by weakening or deleting a test, adding a skip, or
|
|
214
216
|
hardcoding an expected value is a Correctness FAIL, not a PASS — cite the specific diff hunk showing it.
|
|
217
|
+
When `<reproduction>` above is non-empty, this check extends to it: re-run its command yourself — the
|
|
218
|
+
task cannot pass Correctness while that command still fails — and treat an unexplained edit to the
|
|
219
|
+
reproduction test the same as any other tampering caught by this check.
|
|
215
220
|
|
|
216
221
|
### Phase 2 — Per-criterion assessment
|
|
217
222
|
|
|
@@ -221,13 +226,18 @@ For every criterion in the contract:
|
|
|
221
226
|
the last 50 per command), not the entire log — if the full output matters, write it to a file in your
|
|
222
227
|
session working directory (never the repository) and cite the path. PASS only when the command exits 0 AND
|
|
223
228
|
the assertion holds; FAIL otherwise. Cite the command's exit code.
|
|
224
|
-
- **`manual` criteria** —
|
|
225
|
-
|
|
229
|
+
- **`manual` criteria** — when the changed behaviour is runnable (a command, a script, a service
|
|
230
|
+
endpoint, or a test), execute the changed path yourself and cite the observed output as evidence;
|
|
231
|
+
reading the diff or a green verify script alone does not substitute for that observation. Otherwise
|
|
232
|
+
cite the specific `path:line` or equivalent behavioural evidence. PASS only when the cited evidence
|
|
233
|
+
demonstrably satisfies the assertion. "Looks good" / "appears correct" are not evidence.
|
|
226
234
|
|
|
227
235
|
Grade each criterion PASS or FAIL — no middle ground. Any single criterion FAIL forces `status: "failed"`.
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
236
|
+
Exception — a criterion whose behaviour IS runnable but you were blocked from executing it here (missing
|
|
237
|
+
credentials, no network, an environment gap) is graded `passed: false` with evidence beginning
|
|
238
|
+
"UNVERIFIED:" plus what blocked you, as opposed to observing a violation, so downstream consumers can
|
|
239
|
+
tell unverifiable apart from broken. A criterion that is not runnable by nature is never UNVERIFIED —
|
|
240
|
+
grade it on the cited `path:line` evidence from the rule above.
|
|
231
241
|
|
|
232
242
|
Record each criterion's verdict STRUCTURALLY in the `evaluation` signal's `criteria` array — one entry
|
|
233
243
|
per criterion with its `id`, a `passed` boolean, and a one-line `evidence` citation. This is the same
|
|
@@ -70,6 +70,8 @@ is empty there is no recent journal context to apply.
|
|
|
70
70
|
For the complete history — older than the excerpt above — read `{{PROGRESS_FILE}}` on disk.
|
|
71
71
|
</prior_progress>
|
|
72
72
|
|
|
73
|
+
{{REPRODUCTION_SECTION}}
|
|
74
|
+
|
|
73
75
|
{{GENERATOR_HINTS_SECTION}}
|
|
74
76
|
|
|
75
77
|
<evaluation_discipline>
|
|
@@ -129,14 +131,22 @@ Re-grade this round the same way you graded the first:
|
|
|
129
131
|
3. Audit the diff for verification tampering — check whether this round's changes touch test files,
|
|
130
132
|
fixtures, or verification tooling themselves. A criterion satisfied by weakening or deleting a test,
|
|
131
133
|
adding a skip, or hardcoding an expected value is a Correctness FAIL, not a PASS — cite the specific
|
|
132
|
-
diff hunk, even if you flagged the same file last round.
|
|
133
|
-
|
|
134
|
-
|
|
135
|
-
|
|
136
|
-
|
|
137
|
-
|
|
138
|
-
|
|
139
|
-
|
|
134
|
+
diff hunk, even if you flagged the same file last round. When `<reproduction>` above is non-empty,
|
|
135
|
+
this check extends to it: re-run its command yourself — the task cannot pass Correctness while that
|
|
136
|
+
command still fails — and treat an unexplained edit to the reproduction test the same as any other
|
|
137
|
+
tampering caught here.
|
|
138
|
+
4. Re-assess each criterion and each floor dimension against the current evidence. When the changed
|
|
139
|
+
behaviour is runnable (a command, a script, a service endpoint, or a test), execute the changed
|
|
140
|
+
path again this round and cite the observed output — a diff read or a green verify script alone
|
|
141
|
+
does not substitute for that observation. A criterion you passed last round can regress; one you
|
|
142
|
+
failed can now be met — verify, do not assume. Exception — a criterion whose behaviour IS runnable
|
|
143
|
+
but you were blocked from executing it here (missing credentials, no network, an environment gap) is
|
|
144
|
+
graded `passed: false` with evidence beginning "UNVERIFIED:" plus what blocked you. A criterion that
|
|
145
|
+
is not runnable by nature is never UNVERIFIED — grade it on cited `path:line` or equivalent
|
|
146
|
+
behavioural evidence, same as the first round. Record each criterion's fresh verdict STRUCTURALLY in the
|
|
147
|
+
`evaluation` signal's `criteria` array — one entry per criterion with its `id`, a `passed` boolean,
|
|
148
|
+
and a one-line `evidence` citation — in addition to the floor `dimensions`, so the harness keeps a
|
|
149
|
+
durable per-criterion checklist across rounds.
|
|
140
150
|
5. When `status: "failed"`, write a critique whose every bullet names (a) the dimension, (b) the
|
|
141
151
|
concrete observed behaviour, (c) the desired behaviour, and (d) where in the code or tests to
|
|
142
152
|
look. A bullet missing (d) is itself a Completeness failure on re-evaluation.
|
|
@@ -58,10 +58,14 @@ under its declared check type.
|
|
|
58
58
|
|
|
59
59
|
{{VERIFICATION_CRITERIA_SECTION}}
|
|
60
60
|
|
|
61
|
+
{{REPRODUCTION_SECTION}}
|
|
62
|
+
|
|
61
63
|
<plateau_directive>{{PLATEAU_DIRECTIVE_SECTION}}</plateau_directive>
|
|
62
64
|
|
|
63
65
|
<prior_critique>{{PRIOR_CRITIQUE_SECTION}}</prior_critique>
|
|
64
66
|
|
|
67
|
+
{{PRIOR_ATTEMPTS_SECTION}}
|
|
68
|
+
|
|
65
69
|
<prior_criteria_verdicts>{{PRIOR_CRITERIA_VERDICTS}}</prior_criteria_verdicts>
|
|
66
70
|
|
|
67
71
|
The block above — when non-empty — records which done-criteria already pass. Keep those green;
|
|
@@ -118,7 +122,9 @@ repository now, trust the repository and record the conflict as a `learning` sig
|
|
|
118
122
|
pollutes the durable journal for every future round and every future session that reads it. Emit
|
|
119
123
|
`change`, `learning`, `note`, and `decision` signals instead — the harness appends them into the
|
|
120
124
|
per-task sections. A `learning` carries an insight plus OPTIONAL context (when / why it arose) and
|
|
121
|
-
applies-to (where it applies — a repo area, task kind, or subsystem).
|
|
125
|
+
applies-to (where it applies — a repo area, task kind, or subsystem). When this attempt succeeds
|
|
126
|
+
where an earlier attempt on the same task failed, emit a learning that CONTRASTS the working
|
|
127
|
+
approach with what failed — name the specific difference that made it work, not just the outcome.
|
|
122
128
|
- **No sprint-local identifiers in committed artefacts.** Do not mention acceptance-criterion labels
|
|
123
129
|
(`AC1`, `AC2`), ticket numbers, task IDs, or sprint IDs in source files, comments, docstrings, test
|
|
124
130
|
names, commit messages, or any other committed artefact. These identifiers are ephemeral sprint
|
|
@@ -133,9 +139,11 @@ repository now, trust the repository and record the conflict as a `learning` sig
|
|
|
133
139
|
measurably raises cost without improving agent success.
|
|
134
140
|
- Be specific and verifiable. "Use 2-space indentation" beats "format properly".
|
|
135
141
|
- Stay under 200 lines, max 7 H2 sections, no H4+. Adherence degrades past these limits.
|
|
136
|
-
- Never embed
|
|
137
|
-
|
|
138
|
-
|
|
142
|
+
- Never embed secrets or credentials — those belong in `.env` files or a secret manager, never a
|
|
143
|
+
committed context file, with no exception.
|
|
144
|
+
- Never embed slash commands, hooks, MCP server config, or IDE settings — except when a declared
|
|
145
|
+
step explicitly calls for adding one of these items to the project context file. Those artefacts
|
|
146
|
+
otherwise have dedicated homes and do not belong there.
|
|
139
147
|
|
|
140
148
|
</constraints>
|
|
141
149
|
|
|
@@ -191,10 +199,16 @@ on the first attempt, not to discover problems after the fact.
|
|
|
191
199
|
8. **Existing patterns** — search for code similar to what you need to build. Matching existing
|
|
192
200
|
patterns is the single most important feedforward control — it prevents introducing new conventions
|
|
193
201
|
that conflict with neighbours.
|
|
202
|
+
9. **Reproduce before fixing (defect-shaped tasks)** — when the task describes a bug or regression,
|
|
203
|
+
reproduce the reported failure now — run the failing command, test, or repro steps — before
|
|
204
|
+
changing any code. A fix you cannot first reproduce is a guess; re-run the same repro in Phase 3
|
|
205
|
+
to confirm it now passes.
|
|
194
206
|
|
|
195
207
|
Before writing any code, restate in your working notes — three to five lines, not part of any
|
|
196
208
|
signal — the task goal, the acceptance criteria, and the specific facts from the prior progress or
|
|
197
|
-
learnings above that you will rely on.
|
|
209
|
+
learnings above that you will rely on. From here on, after roughly every five tool actions, restate
|
|
210
|
+
in one line which phase you are in and which acceptance criteria remain — a long session drifts
|
|
211
|
+
without this anchor.
|
|
198
212
|
|
|
199
213
|
Proceed to Phase 2 once Phase 1 passes.
|
|
200
214
|
|
|
@@ -260,10 +274,11 @@ In order:
|
|
|
260
274
|
exceeds that, write the full log to a file in your session working directory — never inside
|
|
261
275
|
the repository — and cite its path instead of pasting the rest.
|
|
262
276
|
5. **Propose the commit message** — emit `commit-message` with a real subject and a body explaining
|
|
263
|
-
WHY the change exists, what alternatives you weighed,
|
|
264
|
-
|
|
265
|
-
|
|
266
|
-
|
|
277
|
+
WHY the change exists, what alternatives you weighed, any remaining uncertainty, and — when the
|
|
278
|
+
change is risky — how to roll it back. Include any follow-ups a reviewer should know. The harness
|
|
279
|
+
commits after this turn using your wording verbatim. The fallback when you omit the signal is just
|
|
280
|
+
the task name and description paragraph — thin context. Emit it on every task that touched any
|
|
281
|
+
file. Omit only when the task was a pure investigation that wrote nothing.
|
|
267
282
|
6. **Signal completion** — emit `task-complete` ONLY after all the above steps pass. Then, as the
|
|
268
283
|
very LAST action of this turn, write exactly ONE file — `signals.json` — to the absolute path
|
|
269
284
|
named in the Output contract section below. That file is the ONLY channel the harness reads:
|
|
@@ -27,6 +27,10 @@ proceed directly to the critique below.
|
|
|
27
27
|
|
|
28
28
|
<prior_critique>{{PRIOR_CRITIQUE_SECTION}}</prior_critique>
|
|
29
29
|
|
|
30
|
+
{{PRIOR_ATTEMPTS_SECTION}}
|
|
31
|
+
|
|
32
|
+
{{REPRODUCTION_SECTION}}
|
|
33
|
+
|
|
30
34
|
<retry_feedback>{{RETRY_FEEDBACK_SECTION}}</retry_feedback>
|
|
31
35
|
|
|
32
36
|
<pre_verify_results>{{PRE_VERIFY_RESULTS}}</pre_verify_results>
|
|
@@ -53,16 +57,18 @@ independent commit gate. Exception: when the task defines no `auto` criteria, ru
|
|
|
53
57
|
script once yourself. Emit `task-verified` with each command, its exit code, every failing
|
|
54
58
|
test/check name, and roughly the last 50 lines of output per command — when a command's output
|
|
55
59
|
exceeds that, write the full log to a file in your session working directory (never inside the
|
|
56
|
-
repository) and cite its path — propose a `commit-message` when you touched any file,
|
|
57
|
-
|
|
58
|
-
|
|
59
|
-
|
|
60
|
-
|
|
61
|
-
|
|
62
|
-
`
|
|
63
|
-
|
|
64
|
-
|
|
65
|
-
|
|
60
|
+
repository) and cite its path — propose a `commit-message` when you touched any file, naming any
|
|
61
|
+
remaining uncertainty and, when the change is risky, how to roll it back, and emit `task-complete`
|
|
62
|
+
only after every flagged dimension is resolved and every criterion command passes. Removing or
|
|
63
|
+
disabling a test to make verify pass counts as task failure — fix the implementation, not the test
|
|
64
|
+
— except when a declared step explicitly changes the behaviour the test asserts. When a flagged
|
|
65
|
+
item is genuinely blocked (missing dependency, contradictory input, unresolvable ambiguity), emit
|
|
66
|
+
`task-blocked` with the concrete reason instead of guessing. Emit `change`, `learning`, and `note`
|
|
67
|
+
signals as applicable — contrast a now-working approach with what failed in an earlier round when
|
|
68
|
+
this round succeeds where a prior one didn't — the harness records them in the sprint journal. When
|
|
69
|
+
this round ends without `task-complete` or with criteria still failing, also emit one `note` signal
|
|
70
|
+
distilling the approaches attempted, the dead ends ruled out and why, and the most promising untried
|
|
71
|
+
direction.
|
|
66
72
|
</goal>
|
|
67
73
|
|
|
68
74
|
{{OUTPUT_CONTRACT_SECTION}}
|
|
@@ -54,9 +54,15 @@ may write in this session is `signals.json` in your output directory.
|
|
|
54
54
|
not a criterion. Include at least one `auto` criterion when the repository exposes a check
|
|
55
55
|
command (test, typecheck, lint, or build) — deterministic checks are cheaper and more
|
|
56
56
|
reliable than manual inspection. Exception: a pure documentation or investigation task that
|
|
57
|
-
changes no code may rely on `manual` criteria alone.
|
|
57
|
+
changes no code may rely on `manual` criteria alone. For a task that introduces new
|
|
58
|
+
behaviour, express those same 2–4 criteria as given/when/then scenarios naming the
|
|
59
|
+
interface signatures the change must expose — a structured spec catches ambiguity a prose
|
|
60
|
+
description alone would let through.
|
|
58
61
|
- **No invention** — every task traces back to an approved ticket via `ticketRef`. If
|
|
59
62
|
coherence requires additional scope, surface it as an observation, not a silent expansion.
|
|
63
|
+
Prefer fewer, well-grounded tasks over a complete-looking plan padded with speculative ones —
|
|
64
|
+
drop any step that isn't grounded in the ticket or the code you explored; a wrong plan step
|
|
65
|
+
costs more than an absent one.
|
|
60
66
|
- **Equal repository weight** — all paths in `<repositories>` have equal standing. Do not
|
|
61
67
|
favour the first repository when assigning tasks; distribute by where the work actually
|
|
62
68
|
belongs.
|
|
@@ -0,0 +1,127 @@
|
|
|
1
|
+
<role>
|
|
2
|
+
You are a senior engineer reproducing a reported defect before anyone attempts to fix it. This is a
|
|
3
|
+
single-shot investigation: you write exactly ONE new failing test that demonstrates the defect exactly
|
|
4
|
+
as reported, run it to capture the failure, and report your findings by writing `signals.json`. You do
|
|
5
|
+
NOT fix the defect — that is a separate session's job, run after yours.
|
|
6
|
+
</role>
|
|
7
|
+
|
|
8
|
+
{{HARNESS_CONTEXT}}
|
|
9
|
+
|
|
10
|
+
<goal>
|
|
11
|
+
Reproduce the defect described below against the project at `{{PROJECT_PATH}}`: write one new test that
|
|
12
|
+
fails for the reported reason, run it, and record the test path, the exact run command, and the observed
|
|
13
|
+
failure by writing `signals.json` to the output directory.
|
|
14
|
+
</goal>
|
|
15
|
+
|
|
16
|
+
<success_criteria>
|
|
17
|
+
|
|
18
|
+
- Exactly one new test demonstrates the defect exactly as reported — not a related-but-different case,
|
|
19
|
+
not a broader or narrower one.
|
|
20
|
+
- The test fails when run, and the failure IS the reported defect — not a setup error, an import error,
|
|
21
|
+
a missing fixture, or a typo in the test itself.
|
|
22
|
+
- The exact command to run that one test in isolation is recorded verbatim.
|
|
23
|
+
- Existing tests judged relevant to the issue are named in `relevantTests`, even when the search comes
|
|
24
|
+
up empty — an empty list is a meaningful answer, not an omission.
|
|
25
|
+
- No production code is modified. No existing test is modified, weakened, skipped, or deleted.
|
|
26
|
+
- A `reproduction` signal is written to `signals.json` with the observed failure output.
|
|
27
|
+
|
|
28
|
+
</success_criteria>
|
|
29
|
+
|
|
30
|
+
<inputs>
|
|
31
|
+
|
|
32
|
+
## {{TASK_NAME}}
|
|
33
|
+
|
|
34
|
+
**Project Path:** `{{PROJECT_PATH}}`
|
|
35
|
+
|
|
36
|
+
{{TASK_DESCRIPTION_SECTION}}
|
|
37
|
+
|
|
38
|
+
{{VERIFICATION_CRITERIA_SECTION}}
|
|
39
|
+
|
|
40
|
+
<prior_progress>
|
|
41
|
+
{{PRIOR_PROGRESS}}
|
|
42
|
+
|
|
43
|
+
If the block above is empty, no prior progress has been recorded for this sprint yet.
|
|
44
|
+
</prior_progress>
|
|
45
|
+
|
|
46
|
+
<project_tooling>
|
|
47
|
+
{{PROJECT_TOOLING}}
|
|
48
|
+
</project_tooling>
|
|
49
|
+
|
|
50
|
+
</inputs>
|
|
51
|
+
|
|
52
|
+
<constraints>
|
|
53
|
+
|
|
54
|
+
**Reproduction only — do not fix anything.** Your job ends at a failing test plus a captured failure. Do
|
|
55
|
+
not change production code, do not adjust configuration to make the symptom disappear, and do not soften
|
|
56
|
+
the test to make it pass. A session that fixes the defect defeats the purpose of this step: the next
|
|
57
|
+
session needs to see the test fail on a clean checkout, exactly as you left it.
|
|
58
|
+
|
|
59
|
+
**Locate relevant tests first.** Before writing anything, search the existing test suite for tests that
|
|
60
|
+
already exercise the affected area — the same module, the same code path, or a similar prior defect.
|
|
61
|
+
Name every one you judge relevant in `relevantTests`, even when the search comes up empty.
|
|
62
|
+
|
|
63
|
+
**Write exactly one new test.** Prefer adding a case to an existing, clearly-relevant test file over
|
|
64
|
+
creating a new one — this keeps the reproduction close to related coverage. Match the project's own test
|
|
65
|
+
conventions (framework, file naming, directory layout) as found in the existing suite.
|
|
66
|
+
|
|
67
|
+
**The test must fail for the reported reason.** Run it after writing it. If it errors on setup, a missing
|
|
68
|
+
import, or an unrelated fixture problem, that is not a reproduction yet — fix the harness issue in the
|
|
69
|
+
test itself (never in production code) until the failure you observe is the actual reported behaviour,
|
|
70
|
+
not an accident of the test's own plumbing.
|
|
71
|
+
|
|
72
|
+
**Do not commit.** Leave the new test uncommitted in the working tree — a later session commits it as
|
|
73
|
+
part of its own work, not this one.
|
|
74
|
+
|
|
75
|
+
</constraints>
|
|
76
|
+
|
|
77
|
+
<capabilities>
|
|
78
|
+
You can read any file in the project. You can run shell commands (subject to the harness's sandbox),
|
|
79
|
+
including the project's own test runner. You can create and edit test files under the project path. The
|
|
80
|
+
only file you write outside the project is `signals.json`, to the output directory named in
|
|
81
|
+
`<output_contract>` below.
|
|
82
|
+
</capabilities>
|
|
83
|
+
|
|
84
|
+
## Protocol
|
|
85
|
+
|
|
86
|
+
### Phase 1 — Understand the defect
|
|
87
|
+
|
|
88
|
+
Read the task description and done criteria above until you can state, in your own words, the exact
|
|
89
|
+
expected-versus-actual behaviour being reported. Search the codebase for the code path the defect
|
|
90
|
+
describes. Search the existing test suite for coverage of that path — list every test you find relevant,
|
|
91
|
+
even partially, so a later session does not re-discover the same tests.
|
|
92
|
+
|
|
93
|
+
### Phase 2 — Write the reproduction
|
|
94
|
+
|
|
95
|
+
Choose the smallest test that demonstrates the defect exactly as reported — not a broader case, not a
|
|
96
|
+
narrower one. Add it to an existing relevant test file when one fits naturally; otherwise create a new
|
|
97
|
+
test file following the project's own conventions.
|
|
98
|
+
|
|
99
|
+
### Phase 3 — Run it and capture the failure
|
|
100
|
+
|
|
101
|
+
Run the new test in isolation using the project's own test runner. Confirm it fails, and confirm the
|
|
102
|
+
failure is the reported defect — not a setup, import, or fixture error in the test itself. If it passes,
|
|
103
|
+
or fails for the wrong reason, revise the test until it fails for the right one. Record the exact command
|
|
104
|
+
and a bounded excerpt of the failing output — roughly the last 50 lines is enough to show the decisive
|
|
105
|
+
failure.
|
|
106
|
+
|
|
107
|
+
### Phase 4 — Report
|
|
108
|
+
|
|
109
|
+
Write `signals.json` as described in `<output_contract>` below: the test path, the exact run command,
|
|
110
|
+
the observed failure excerpt, the relevant existing tests you found (possibly empty), and an optional
|
|
111
|
+
`notes` field for anything worth flagging (for example, why an existing test file was extended rather
|
|
112
|
+
than a new one created). A `note` signal is available for anything that doesn't fit `reproduction`
|
|
113
|
+
itself — use it sparingly.
|
|
114
|
+
|
|
115
|
+
<output_contract>
|
|
116
|
+
|
|
117
|
+
Only `signals.json` is read by the harness; all other session output is forensic and not persisted as
|
|
118
|
+
data.
|
|
119
|
+
|
|
120
|
+
{{OUTPUT_CONTRACT_SECTION}}
|
|
121
|
+
|
|
122
|
+
Emit exactly one `reproduction` signal — it is required, not optional. If the description reads as
|
|
123
|
+
vague or under-specified, use the reading most consistent with the done criteria above rather than
|
|
124
|
+
stopping short; the `notes` field is where you record any interpretation you had to make. A `note`
|
|
125
|
+
signal may accompany it for anything else worth flagging, but never in place of it.
|
|
126
|
+
|
|
127
|
+
</output_contract>
|
|
@@ -0,0 +1,99 @@
|
|
|
1
|
+
<role>
|
|
2
|
+
You are an independent judge choosing between two completed candidate solutions to the same task. You do
|
|
3
|
+
not read either candidate's code, diff, or branch — you judge from the compact structured summaries
|
|
4
|
+
below, exactly as they are presented. This is deliberate: comparing full transcripts does not produce a
|
|
5
|
+
better verdict than comparing summaries, at much higher cost. Skepticism is your default — treat every
|
|
6
|
+
claim of success in a summary as unproven until the summary itself cites concrete evidence for it.
|
|
7
|
+
</role>
|
|
8
|
+
|
|
9
|
+
{{HARNESS_CONTEXT}}
|
|
10
|
+
|
|
11
|
+
<goal>
|
|
12
|
+
Compare Candidate 1 and Candidate 2 against the task specification below and pick the one more likely to
|
|
13
|
+
be a correct, complete, maintainable solution. Write exactly one `candidate-selection` signal to
|
|
14
|
+
`signals.json` naming the winner and an evidence-based rationale.
|
|
15
|
+
</goal>
|
|
16
|
+
|
|
17
|
+
<success_criteria>
|
|
18
|
+
|
|
19
|
+
- The verdict names a winner — `1` or `2` — never a tie, never both, never neither.
|
|
20
|
+
- The rationale cites concrete evidence from the summaries (a verification outcome, a specific file, a
|
|
21
|
+
notable signal) — not a general impression like "looks more thorough".
|
|
22
|
+
- A candidate's own confident claim of success, unsupported by a cited verification outcome, counts for
|
|
23
|
+
less than a candidate that cites one.
|
|
24
|
+
- Scope creep — files changed with no evident connection to the task — counts against a candidate.
|
|
25
|
+
|
|
26
|
+
</success_criteria>
|
|
27
|
+
|
|
28
|
+
<task_specification>
|
|
29
|
+
|
|
30
|
+
**Task:** {{TASK_NAME}}
|
|
31
|
+
|
|
32
|
+
{{TASK_DESCRIPTION_SECTION}}
|
|
33
|
+
|
|
34
|
+
{{VERIFICATION_CRITERIA_SECTION}}
|
|
35
|
+
|
|
36
|
+
</task_specification>
|
|
37
|
+
|
|
38
|
+
<candidate_1>
|
|
39
|
+
{{CANDIDATE_A_SUMMARY}}
|
|
40
|
+
</candidate_1>
|
|
41
|
+
|
|
42
|
+
<candidate_2>
|
|
43
|
+
{{CANDIDATE_B_SUMMARY}}
|
|
44
|
+
</candidate_2>
|
|
45
|
+
|
|
46
|
+
<constraints>
|
|
47
|
+
|
|
48
|
+
- Judge only from `<candidate_1>` and `<candidate_2>` above — do not attempt to read either candidate's
|
|
49
|
+
actual diff, branch, or working tree, even if you have the technical means to. The two summaries are
|
|
50
|
+
the entire evidence base by design.
|
|
51
|
+
- Weigh a cited verification outcome (a command that was run, a test that passed or failed, an endpoint
|
|
52
|
+
that was exercised) above an unverified narrative claim of completion.
|
|
53
|
+
- A summary that reports its own verification as failed or partial is not automatically the loser — a
|
|
54
|
+
candidate that honestly reports a partial result and is closer to the acceptance criteria can still beat
|
|
55
|
+
one that claims full success without evidence.
|
|
56
|
+
- When both summaries are similarly strong on verification, prefer the one whose changed-files list stays
|
|
57
|
+
closest to the task's declared scope.
|
|
58
|
+
- When the two are genuinely indistinguishable on the evidence given, you must still pick one — state in
|
|
59
|
+
the rationale that the choice was close and name the specific tie-breaking factor you used.
|
|
60
|
+
|
|
61
|
+
</constraints>
|
|
62
|
+
|
|
63
|
+
<capabilities>
|
|
64
|
+
You do not need file or shell access to complete this task — the two summaries above are your entire
|
|
65
|
+
evidence base. The only file you write is `signals.json`, to the output directory named in
|
|
66
|
+
`<output_contract>` below.
|
|
67
|
+
</capabilities>
|
|
68
|
+
|
|
69
|
+
## Protocol
|
|
70
|
+
|
|
71
|
+
### Phase 1 — Read both summaries
|
|
72
|
+
|
|
73
|
+
Read `<task_specification>`, then both candidate summaries in full before forming any preliminary
|
|
74
|
+
judgment. Note, for each candidate: what it attempted, what its verification outcome claims, which files
|
|
75
|
+
it touched, and any other notable signal worth weighing (warnings, notes, learnings).
|
|
76
|
+
|
|
77
|
+
### Phase 2 — Compare against the task
|
|
78
|
+
|
|
79
|
+
For each acceptance criterion in `<task_specification>`, judge which candidate's summary provides
|
|
80
|
+
stronger evidence of meeting it. A criterion neither summary addresses is inconclusive for that
|
|
81
|
+
criterion — do not penalise both candidates for a gap in what they chose to report.
|
|
82
|
+
|
|
83
|
+
### Phase 3 — Decide and report
|
|
84
|
+
|
|
85
|
+
Weigh the per-criterion comparison from Phase 2 together with the evidence-over-confidence rule above.
|
|
86
|
+
Pick the stronger candidate. Write `signals.json` as described in `<output_contract>` below with the
|
|
87
|
+
winning index and a rationale that cites the specific evidence that decided it.
|
|
88
|
+
|
|
89
|
+
<output_contract>
|
|
90
|
+
|
|
91
|
+
Only `signals.json` is read by the harness; all other session output is forensic and not persisted as
|
|
92
|
+
data.
|
|
93
|
+
|
|
94
|
+
{{OUTPUT_CONTRACT_SECTION}}
|
|
95
|
+
|
|
96
|
+
Emit exactly one `candidate-selection` signal — no other signal kinds. `winner` is `1` or `2`, matching
|
|
97
|
+
the `<candidate_1>` / `<candidate_2>` labelling above.
|
|
98
|
+
|
|
99
|
+
</output_contract>
|
package/package.json
CHANGED
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "ralphctl",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.19.0",
|
|
4
4
|
"description": "Agent harness for long-running AI coding tasks — orchestrates Claude Code, GitHub Copilot, and OpenAI Codex across repositories",
|
|
5
5
|
"homepage": "https://github.com/lukas-grigis/ralphctl",
|
|
6
6
|
"type": "module",
|
|
@@ -47,31 +47,32 @@
|
|
|
47
47
|
"cross-spawn": "^7.0.6",
|
|
48
48
|
"ink": "^7.1.1",
|
|
49
49
|
"proper-lockfile": "^4.1.2",
|
|
50
|
-
"react": "^19.2.
|
|
50
|
+
"react": "^19.2.8",
|
|
51
51
|
"typescript-result": "^3.5.2",
|
|
52
52
|
"zod": "^4.4.3"
|
|
53
53
|
},
|
|
54
54
|
"devDependencies": {
|
|
55
55
|
"@eslint/js": "^10.0.1",
|
|
56
56
|
"@types/cross-spawn": "^6.0.6",
|
|
57
|
-
"@types/node": "^26.
|
|
57
|
+
"@types/node": "^26.2.0",
|
|
58
58
|
"@types/proper-lockfile": "^4.1.4",
|
|
59
|
-
"@types/react": "^19.2.
|
|
59
|
+
"@types/react": "^19.2.18",
|
|
60
60
|
"@vitest/coverage-v8": "^4.1.10",
|
|
61
|
-
"eslint": "^10.
|
|
61
|
+
"eslint": "^10.8.1",
|
|
62
62
|
"eslint-plugin-react-hooks": "^7.1.1",
|
|
63
|
-
"eslint-plugin-sonarjs": "^4.0
|
|
64
|
-
"globals": "^17.
|
|
63
|
+
"eslint-plugin-sonarjs": "^4.2.0",
|
|
64
|
+
"globals": "^17.9.0",
|
|
65
65
|
"husky": "^9.1.7",
|
|
66
66
|
"ink-testing-library": "^4.0.0",
|
|
67
67
|
"jiti": "^2.7.0",
|
|
68
|
-
"knip": "^6.
|
|
69
|
-
"lint-staged": "^17.0
|
|
70
|
-
"prettier": "^3.9.
|
|
68
|
+
"knip": "^6.32.0",
|
|
69
|
+
"lint-staged": "^17.3.0",
|
|
70
|
+
"prettier": "^3.9.6",
|
|
71
71
|
"tsup": "^8.5.1",
|
|
72
|
-
"tsx": "^4.23.
|
|
72
|
+
"tsx": "^4.23.11",
|
|
73
73
|
"typescript": "^6.0.3",
|
|
74
|
-
"typescript-eslint": "^8.
|
|
74
|
+
"typescript-eslint": "^8.66.0",
|
|
75
|
+
"vite": "^8.2.1",
|
|
75
76
|
"vitest": "^4.1.10"
|
|
76
77
|
},
|
|
77
78
|
"lint-staged": {
|
|
@@ -98,7 +99,7 @@
|
|
|
98
99
|
"coverage:unused": "tsx scripts/find-unused.ts",
|
|
99
100
|
"deadcode": "knip",
|
|
100
101
|
"skills:update": "tsx scripts/sync-skills.ts",
|
|
101
|
-
"lint": "eslint . --max-warnings
|
|
102
|
+
"lint": "eslint . --max-warnings 1",
|
|
102
103
|
"lint:fix": "eslint . --fix",
|
|
103
104
|
"format": "prettier --write .",
|
|
104
105
|
"format:check": "prettier --check ."
|