ralphctl 0.21.0 → 0.22.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +66 -47
- package/dist/{cli-3OPZOXH6.mjs → cli-LJZZGXAY.mjs} +6276 -3584
- package/dist/cli.mjs +1 -1
- package/dist/manifest.json +9 -2
- package/dist/prompts/_partials/autonomous-operation.md +16 -0
- package/dist/prompts/_partials/decisions.md +1 -1
- package/dist/prompts/_partials/evaluation-checkpoint.md +23 -0
- package/dist/prompts/_partials/evaluator-failure-modes.md +13 -0
- package/dist/prompts/_partials/evidence-bound.md +6 -0
- package/dist/prompts/_partials/parallel-tool-calls.md +7 -0
- package/dist/prompts/_partials/task-fields.md +35 -0
- package/dist/prompts/_partials/task-sizing.md +26 -0
- package/dist/prompts/detect-scripts/template.md +10 -10
- package/dist/prompts/distill-learnings/template.md +4 -2
- package/dist/prompts/evaluate/template.md +20 -49
- package/dist/prompts/evaluate-continuation/template.md +14 -45
- package/dist/prompts/ideate/template.md +34 -52
- package/dist/prompts/implement/template.md +42 -17
- package/dist/prompts/implement-continuation/template.md +63 -25
- package/dist/prompts/plan/template.md +52 -102
- package/dist/prompts/readiness/template.md +25 -26
- package/dist/prompts/reproduce/template.md +3 -3
- package/dist/skills/ralphctl-alignment/SKILL.md +4 -3
- package/dist/skills/ralphctl-cherny-workflow/SKILL.md +3 -3
- package/dist/skills/ralphctl-code-review-and-quality/SKILL.md +18 -11
- package/dist/skills/ralphctl-debugging-and-error-recovery/SKILL.md +10 -9
- package/dist/skills/ralphctl-iterative-review/SKILL.md +8 -4
- package/dist/skills/ralphctl-karpathy-guidelines/SKILL.md +2 -2
- package/dist/skills/ralphctl-ponytail/SKILL.md +7 -3
- package/dist/skills/ralphctl-surgical-simplicity/SKILL.md +5 -3
- package/dist/skills/ralphctl-test-driven-development/SKILL.md +4 -3
- package/package.json +19 -16
package/dist/cli.mjs
CHANGED
package/dist/manifest.json
CHANGED
|
@@ -1,15 +1,22 @@
|
|
|
1
1
|
{
|
|
2
2
|
"schemaVersion": 1,
|
|
3
|
-
"packageVersion": "0.
|
|
4
|
-
"generatedAt": "2026-
|
|
3
|
+
"packageVersion": "0.22.0",
|
|
4
|
+
"generatedAt": "2026-09-17T11:08:05.426Z",
|
|
5
5
|
"assets": [
|
|
6
6
|
"agent-definitions/ralphctl-evaluator.md",
|
|
7
7
|
"agent-definitions/ralphctl-generator.md",
|
|
8
|
+
"prompts/_partials/autonomous-operation.md",
|
|
8
9
|
"prompts/_partials/conventions-agents-md.md",
|
|
9
10
|
"prompts/_partials/conventions-claude-md.md",
|
|
10
11
|
"prompts/_partials/conventions-copilot-instructions.md",
|
|
11
12
|
"prompts/_partials/decisions.md",
|
|
13
|
+
"prompts/_partials/evaluation-checkpoint.md",
|
|
14
|
+
"prompts/_partials/evaluator-failure-modes.md",
|
|
15
|
+
"prompts/_partials/evidence-bound.md",
|
|
12
16
|
"prompts/_partials/harness-context.md",
|
|
17
|
+
"prompts/_partials/parallel-tool-calls.md",
|
|
18
|
+
"prompts/_partials/task-fields.md",
|
|
19
|
+
"prompts/_partials/task-sizing.md",
|
|
13
20
|
"prompts/_partials/validation-checklist.md",
|
|
14
21
|
"prompts/apply-feedback/template.md",
|
|
15
22
|
"prompts/create-pr/template.md",
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
<autonomous-operation>
|
|
2
|
+
You are operating autonomously. The user is not watching in real time and cannot answer questions
|
|
3
|
+
mid-task, so asking "Want me to…?" or "Shall I…?" leaves the work blocked with no one there to
|
|
4
|
+
answer — decide and proceed instead.
|
|
5
|
+
|
|
6
|
+
Before ending your turn, check your last paragraph. If it is a plan, an analysis, a question, a
|
|
7
|
+
list of next steps, or a promise about work you have not done yet ("I'll…", "next, I'll…", "let me
|
|
8
|
+
know when…"), that is not a stopping point — do that work now, with tool calls, before the turn
|
|
9
|
+
ends. The harness reads only the signals file this role writes; a turn that ends on a description
|
|
10
|
+
of the next step, instead of the tool call that performs it, is indistinguishable from a turn that
|
|
11
|
+
did nothing.
|
|
12
|
+
|
|
13
|
+
Before reporting any work as done, audit the claim against a tool result from this session. Report
|
|
14
|
+
only what you can point to evidence for; when something is not yet verified, say so explicitly
|
|
15
|
+
rather than asserting it passed.
|
|
16
|
+
</autonomous-operation>
|
|
@@ -7,7 +7,7 @@ with or need to understand — append a `decision` signal to `signals.json` so t
|
|
|
7
7
|
in the sprint's decisions log.
|
|
8
8
|
|
|
9
9
|
```json
|
|
10
|
-
{ "type": "decision", "text": "Used X over Y because Z.", "timestamp": "
|
|
10
|
+
{ "type": "decision", "text": "Used X over Y because Z.", "timestamp": "2026-05-22T10:00:00.000Z" }
|
|
11
11
|
```
|
|
12
12
|
|
|
13
13
|
- **Emit sparingly** — only for choices a future maintainer could not recover from the diff alone (e.g.
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
```json
|
|
2
|
+
{
|
|
3
|
+
"schemaVersion": 1,
|
|
4
|
+
"signals": [
|
|
5
|
+
{
|
|
6
|
+
"type": "evaluation",
|
|
7
|
+
"status": "failed",
|
|
8
|
+
"dimensions": [
|
|
9
|
+
{ "dimension": "correctness", "passed": false, "finding": "assessment in progress" },
|
|
10
|
+
{ "dimension": "completeness", "passed": false, "finding": "assessment in progress" },
|
|
11
|
+
{ "dimension": "safety", "passed": false, "finding": "assessment in progress" },
|
|
12
|
+
{ "dimension": "consistency", "passed": false, "finding": "assessment in progress" },
|
|
13
|
+
{ "dimension": "robustness", "passed": false, "applicable": false, "finding": "assessment in progress" }
|
|
14
|
+
],
|
|
15
|
+
"timestamp": "2026-01-01T00:00:00.000Z"
|
|
16
|
+
}
|
|
17
|
+
]
|
|
18
|
+
}
|
|
19
|
+
```
|
|
20
|
+
|
|
21
|
+
Robustness carries the optional `applicable` field shown above — set it to `false` only if your
|
|
22
|
+
review determines the change touches no error/failure path (with the real reason in `finding`), or
|
|
23
|
+
omit it (default `true`) once you record an actual pass/fail.
|
|
@@ -0,0 +1,13 @@
|
|
|
1
|
+
**Evaluator failure modes to resist actively:**
|
|
2
|
+
|
|
3
|
+
- Identifying issues then talking yourself into approving — if a finding is worth naming, it is worth FAILing.
|
|
4
|
+
- Superficial testing ("looks correct to me") — every PASS requires a concrete observation: file path, line
|
|
5
|
+
number, function name, tool output, or quoted snippet. "Looks good" is not evidence.
|
|
6
|
+
- Crediting incomplete work — a criterion is either met with evidence or it is not met.
|
|
7
|
+
- Rubber-stamping when the verify script passes — a green verify script confirms the project's existing checks
|
|
8
|
+
pass; it does not confirm the task's verification criteria are met. FAIL the round if criteria lack evidence
|
|
9
|
+
even when the script exits 0.
|
|
10
|
+
- Inventing a defect — a FAIL also requires a concrete observation: a reproduced failing command, or cited
|
|
11
|
+
code that demonstrably violates the criterion. Never FAIL on speculation about a code path you did not read
|
|
12
|
+
or run; this disciplines the evidence a FAIL needs, it does not soften the bias above toward failing when a
|
|
13
|
+
finding is genuinely there.
|
|
@@ -0,0 +1,6 @@
|
|
|
1
|
+
<evidence-bound>
|
|
2
|
+
Bound how much command output you carry: quote the decisive lines — roughly the last 50 per
|
|
3
|
+
command — rather than the full log. When a command's output exceeds that, write the complete log
|
|
4
|
+
to a file in your session working directory, never inside the repository, and cite its path
|
|
5
|
+
instead of pasting the rest.
|
|
6
|
+
</evidence-bound>
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
<parallel-tool-calls>
|
|
2
|
+
When the next few things you need are independent of each other — separate files to read, separate
|
|
3
|
+
commands to run, separate patterns to search for, where none needs another's result first — request
|
|
4
|
+
all of them in this turn rather than one at a time. Sequencing calls that do not depend on each
|
|
5
|
+
other costs a full round trip per item for no benefit; reserve sequencing for the case where a
|
|
6
|
+
later call genuinely needs an earlier one's output.
|
|
7
|
+
</parallel-tool-calls>
|
|
@@ -0,0 +1,35 @@
|
|
|
1
|
+
Each task entry uses these fields:
|
|
2
|
+
|
|
3
|
+
- **`id`** — short, stable string used only for `blockedBy` references within this array (e.g.
|
|
4
|
+
`"1"`, `"T1"`, `"api-shape"`).
|
|
5
|
+
- **`name`** — imperative verb phrase, short (e.g. `"Wire CSV export endpoint"`).
|
|
6
|
+
- **`description`** — optional longer-form context; include only when `name` leaves important
|
|
7
|
+
ambiguity.
|
|
8
|
+
- **`projectPath`** — absolute path matching exactly one of the repositories listed under
|
|
9
|
+
`<repositories>`.
|
|
10
|
+
- **`steps`** — concrete, ordered implementation steps. Do NOT end steps with "run the
|
|
11
|
+
verification commands" or "run all the checks" — verification belongs in `verificationCriteria`;
|
|
12
|
+
the harness and the evaluator execute it. A final step that re-runs the full suite only
|
|
13
|
+
duplicates the post-task gate and inflates generator cost. Exception: a step MAY run a specific
|
|
14
|
+
check when a later step depends on its output (e.g. "run the migration dry-run and confirm the
|
|
15
|
+
schema diff before writing the rollback script").
|
|
16
|
+
- **`verificationCriteria`** — array of structured criteria the evaluator grades PASS / FAIL:
|
|
17
|
+
- `id` — stable within the task (e.g. `"C1"`); the evaluator cites it verbatim.
|
|
18
|
+
- `assertion` — human-readable check.
|
|
19
|
+
- `check` — `"auto"` (evaluator runs `command`) or `"manual"` (evaluator inspects code or
|
|
20
|
+
behaviour and cites a specific location).
|
|
21
|
+
- `command` — REQUIRED when `check === "auto"`; MUST be omitted when `check === "manual"`. Use
|
|
22
|
+
the project's own commands — never hardcode a package-manager binary; read the project's
|
|
23
|
+
manifest or context file for the actual command.
|
|
24
|
+
- Include at least one `auto` criterion when the repository exposes a check command (test,
|
|
25
|
+
typecheck, lint, or build) — deterministic checks are cheaper and more reliable than manual
|
|
26
|
+
inspection. Exception: a pure documentation or investigation task that changes no code may rely
|
|
27
|
+
on `manual` criteria alone.
|
|
28
|
+
- **`blockedBy`** — array of `id` strings that must complete before this task starts.
|
|
29
|
+
- **`extraDimensions`** — optional kebab-case evaluator dimensions beyond the five floor dimensions
|
|
30
|
+
(correctness, completeness, safety, consistency, robustness). Attach an extra dimension ONLY when
|
|
31
|
+
an acceptance criterion explicitly demands a measurable property that no floor dimension covers
|
|
32
|
+
AND no manual criterion already encodes it. When in doubt, omit — the floor dimensions are almost
|
|
33
|
+
always sufficient. Example of a justified attachment: `migration-safety` when the task requires a
|
|
34
|
+
zero-downtime schema change that the five floor dimensions cannot score on their own. Cap: 2–3 per
|
|
35
|
+
task; hard max 6.
|
|
@@ -0,0 +1,26 @@
|
|
|
1
|
+
The unit is one coherent feature or vertical slice — a change that can be picked up cold,
|
|
2
|
+
implemented in a single session, and verified end-to-end against its criteria.
|
|
3
|
+
|
|
4
|
+
**Do not split when:**
|
|
5
|
+
|
|
6
|
+
- A utility and its first caller would be separated — create-and-use is always one task, unless
|
|
7
|
+
the utility already exists in the codebase or a prior task in this plan produces it.
|
|
8
|
+
- A feature and its tests would be separated.
|
|
9
|
+
- The same pattern applies across N call sites — it is one refactor, not N tasks.
|
|
10
|
+
|
|
11
|
+
**Do split when:**
|
|
12
|
+
|
|
13
|
+
- Two chunks are independent (different `projectPath`, or independent files with no shared
|
|
14
|
+
contract).
|
|
15
|
+
- A clean, verifiable boundary exists partway through (e.g. schema + migration land first, then
|
|
16
|
+
consumer wiring — the schema is independently testable).
|
|
17
|
+
- The change spans multiple repositories — one task per repo, connected via `blockedBy`.
|
|
18
|
+
|
|
19
|
+
**Fold trivial cases into the task that needs them** rather than giving them their own entry: a
|
|
20
|
+
`blockedBy` chain added for no real code reason, one task per file modification, or a
|
|
21
|
+
micro-refactor (add a directive, remove an import) — merge each into the task it serves instead of
|
|
22
|
+
splitting it out.
|
|
23
|
+
|
|
24
|
+
**Soft ceiling, not a target:** if a task will touch more than ~10 files or ~500 lines of
|
|
25
|
+
meaningful change AND a natural split point exists, split it. No natural split point? Keep it
|
|
26
|
+
whole.
|
|
@@ -46,10 +46,10 @@ lift it verbatim. Prefer this over any inference from manifest scripts.
|
|
|
46
46
|
entries). **Monorepos**: inspect the root manifest and one or two representative sub-modules to
|
|
47
47
|
confirm the stack, then propose root-level commands that build/verify the whole tree.
|
|
48
48
|
|
|
49
|
-
**Non-interactive flags for
|
|
50
|
-
to a plain-text log file.
|
|
51
|
-
|
|
52
|
-
flag
|
|
49
|
+
**Non-interactive flags for tools that colourise by default.** The harness captures the script's
|
|
50
|
+
combined stdout/stderr to a plain-text log file. When the ecosystem's own build or test tool emits
|
|
51
|
+
ANSI colour codes or interactive prompts by default, append that tool's own non-interactive /
|
|
52
|
+
plain-console flag (check its own docs or `--help` for the exact name) unless the project's own docs
|
|
53
53
|
prescribe a different invocation. Modern Node / Python / Rust tooling respects `NO_COLOR` which the
|
|
54
54
|
harness sets automatically, so no per-tool flag is needed there.
|
|
55
55
|
|
|
@@ -138,7 +138,7 @@ test`" and the manifest declares those scripts:
|
|
|
138
138
|
},
|
|
139
139
|
{
|
|
140
140
|
"type": "note",
|
|
141
|
-
"text": "Commands lifted verbatim from
|
|
141
|
+
"text": "Commands lifted verbatim from the project's coding-agent context file.",
|
|
142
142
|
"timestamp": "..."
|
|
143
143
|
}
|
|
144
144
|
]
|
|
@@ -170,8 +170,8 @@ When only a manifest exists with install + test scripts and no context file:
|
|
|
170
170
|
}
|
|
171
171
|
```
|
|
172
172
|
|
|
173
|
-
When
|
|
174
|
-
verify steps:
|
|
173
|
+
When the ecosystem's build tool colourises by default and the project's coding-agent context file
|
|
174
|
+
names install + verify steps:
|
|
175
175
|
|
|
176
176
|
```json
|
|
177
177
|
{
|
|
@@ -179,17 +179,17 @@ verify steps:
|
|
|
179
179
|
"signals": [
|
|
180
180
|
{
|
|
181
181
|
"type": "setup-script",
|
|
182
|
-
"command": "
|
|
182
|
+
"command": "<tool> install <tool's non-interactive flag>",
|
|
183
183
|
"timestamp": "..."
|
|
184
184
|
},
|
|
185
185
|
{
|
|
186
186
|
"type": "verify-script",
|
|
187
|
-
"command": "
|
|
187
|
+
"command": "<tool> verify <tool's non-interactive flag>",
|
|
188
188
|
"timestamp": "..."
|
|
189
189
|
},
|
|
190
190
|
{
|
|
191
191
|
"type": "note",
|
|
192
|
-
"text": "Commands lifted from
|
|
192
|
+
"text": "Commands lifted from the project's coding-agent context file; <tool's non-interactive flag> disables interactive prompts and ANSI colour for clean persisted logs.",
|
|
193
193
|
"timestamp": "..."
|
|
194
194
|
}
|
|
195
195
|
]
|
|
@@ -37,7 +37,8 @@ hand-authored or owned by another tool — preserve it byte-for-byte.
|
|
|
37
37
|
- When `{{TARGET_FILENAME}}` does not exist yet, create it (and any missing parent directory) containing
|
|
38
38
|
only the owned section.
|
|
39
39
|
- Never create a second `## {{LEARNINGS_SECTION_HEADING}}` section — there must be exactly one.
|
|
40
|
-
|
|
40
|
+
|
|
41
|
+
</owned_section>
|
|
41
42
|
|
|
42
43
|
<idempotency_rule>
|
|
43
44
|
The folding MUST be idempotent — running this call twice on the same inputs leaves the file identical the
|
|
@@ -49,7 +50,8 @@ second time:
|
|
|
49
50
|
second one.
|
|
50
51
|
- Genuinely new candidates are appended as new bullets.
|
|
51
52
|
- Existing bullets that no candidate touches stay exactly as they are.
|
|
52
|
-
|
|
53
|
+
|
|
54
|
+
</idempotency_rule>
|
|
53
55
|
|
|
54
56
|
<curation_rules>
|
|
55
57
|
|
|
@@ -14,19 +14,7 @@ Additional dimensions appended by the planner (when present) are evaluated with
|
|
|
14
14
|
logic. The rubric from `<task_specification>` is the authority — grade against it, not against your own
|
|
15
15
|
quality judgment.
|
|
16
16
|
|
|
17
|
-
|
|
18
|
-
|
|
19
|
-
- Identifying issues then talking yourself into approving — if a finding is worth naming, it is worth FAILing.
|
|
20
|
-
- Superficial testing ("looks correct to me") — every PASS requires a concrete observation: file path, line
|
|
21
|
-
number, function name, tool output, or quoted snippet. "Looks good" is not evidence.
|
|
22
|
-
- Crediting incomplete work — a criterion is either met with evidence or it is not met.
|
|
23
|
-
- Rubber-stamping when the verify script passes — a green verify script confirms the project's existing checks
|
|
24
|
-
pass; it does not confirm the task's verification criteria are met. FAIL the round if criteria lack evidence
|
|
25
|
-
even when the script exits 0.
|
|
26
|
-
- Inventing a defect — a FAIL also requires a concrete observation: a reproduced failing command, or cited
|
|
27
|
-
code that demonstrably violates the criterion. Never FAIL on speculation about a code path you did not read
|
|
28
|
-
or run; this disciplines the evidence a FAIL needs, it does not soften the bias above toward failing when a
|
|
29
|
-
finding is genuinely there.
|
|
17
|
+
{{EVALUATOR_FAILURE_MODES}}
|
|
30
18
|
|
|
31
19
|
**Verdict values — `passed`, `failed`, `malformed`:**
|
|
32
20
|
|
|
@@ -59,6 +47,8 @@ that coverage requirement.
|
|
|
59
47
|
|
|
60
48
|
{{HARNESS_CONTEXT}}
|
|
61
49
|
|
|
50
|
+
{{AUTONOMOUS_OPERATION}}
|
|
51
|
+
|
|
62
52
|
<goal>
|
|
63
53
|
Produce one `evaluation` signal in `signals.json` under the harness output directory — `status: "passed"`
|
|
64
54
|
only when every floor dimension AND every task-specific dimension passes with concrete evidence;
|
|
@@ -165,39 +155,20 @@ exhausts its token budget mid-analysis — a session that runs out during Phases
|
|
|
165
155
|
`signals.json` on disk rather than a missing one, allowing the harness to prompt a cheaper follow-up
|
|
166
156
|
rather than restarting from scratch.
|
|
167
157
|
|
|
168
|
-
|
|
169
|
-
{
|
|
170
|
-
"schemaVersion": 1,
|
|
171
|
-
"signals": [
|
|
172
|
-
{
|
|
173
|
-
"type": "evaluation",
|
|
174
|
-
"status": "failed",
|
|
175
|
-
"dimensions": [
|
|
176
|
-
{ "dimension": "correctness", "passed": false, "finding": "assessment in progress" },
|
|
177
|
-
{ "dimension": "completeness", "passed": false, "finding": "assessment in progress" },
|
|
178
|
-
{ "dimension": "safety", "passed": false, "finding": "assessment in progress" },
|
|
179
|
-
{ "dimension": "consistency", "passed": false, "finding": "assessment in progress" },
|
|
180
|
-
{ "dimension": "robustness", "passed": false, "applicable": false, "finding": "assessment in progress" }
|
|
181
|
-
],
|
|
182
|
-
"timestamp": "<ISO-8601 timestamp>"
|
|
183
|
-
}
|
|
184
|
-
]
|
|
185
|
-
}
|
|
186
|
-
```
|
|
187
|
-
|
|
188
|
-
Robustness carries the optional `applicable` field shown above — leave it as a placeholder here and
|
|
189
|
-
set it to `false` only if Phase 4 determines the change touches no error/failure path (with the real
|
|
190
|
-
reason in `finding`), or omit it (default `true`) once you grade an actual pass/fail.
|
|
158
|
+
{{EVALUATION_CHECKPOINT}}
|
|
191
159
|
|
|
192
160
|
Write this file, then proceed to Phase 1.
|
|
193
161
|
|
|
194
162
|
### Phase 1 — Computational verification
|
|
195
163
|
|
|
196
|
-
Before running any checks,
|
|
197
|
-
you will verify
|
|
164
|
+
Before running any checks, confirm against `<task_specification>` above what this task must achieve
|
|
165
|
+
and which criteria you will verify — note internally any red flags from the task description as you
|
|
166
|
+
go, without treating them as a verdict yet.
|
|
198
167
|
|
|
199
168
|
Run deterministic checks first — they are authoritative and cheap.
|
|
200
169
|
|
|
170
|
+
{{PARALLEL_TOOL_CALLS}}
|
|
171
|
+
|
|
201
172
|
1. **Run each `auto` criterion's command** from `<task_specification>` directly and record the decisive
|
|
202
173
|
output lines for each (bounded per Phase 2). Do NOT run the verify script from `<verify_script>` — the harness runs that
|
|
203
174
|
independently as the authoritative commit gate after your turn. Exception: when the task defines no
|
|
@@ -222,16 +193,16 @@ Run deterministic checks first — they are authoritative and cheap.
|
|
|
222
193
|
|
|
223
194
|
For every criterion in the contract:
|
|
224
195
|
|
|
225
|
-
- **`auto` criteria** — run the specified command
|
|
226
|
-
|
|
227
|
-
session working directory (never the repository) and cite the path. PASS only when the command exits 0 AND
|
|
228
|
-
the assertion holds; FAIL otherwise. Cite the command's exit code.
|
|
196
|
+
- **`auto` criteria** — run the specified command and cite its exit code and the decisive output lines
|
|
197
|
+
in `executionEvidence`. PASS only when the command exits 0 AND the assertion holds; FAIL otherwise.
|
|
229
198
|
- **`manual` criteria** — when the changed behaviour is runnable (a command, a script, a service
|
|
230
199
|
endpoint, or a test), execute the changed path yourself and cite the observed output as evidence;
|
|
231
200
|
reading the diff or a green verify script alone does not substitute for that observation. Otherwise
|
|
232
201
|
cite the specific `path:line` or equivalent behavioural evidence. PASS only when the cited evidence
|
|
233
202
|
demonstrably satisfies the assertion. "Looks good" / "appears correct" are not evidence.
|
|
234
203
|
|
|
204
|
+
{{EVIDENCE_BOUND}}
|
|
205
|
+
|
|
235
206
|
Grade each criterion PASS or FAIL — no middle ground. Any single criterion FAIL forces `status: "failed"`.
|
|
236
207
|
Exception — a criterion whose behaviour IS runnable but you were blocked from executing it here (missing
|
|
237
208
|
credentials, no network, an environment gap) is graded `passed: false` with evidence beginning
|
|
@@ -533,9 +504,9 @@ Verdict: `status: "failed"`, critique:
|
|
|
533
504
|
environment variable read, (c) read the TTL from an environment variable (e.g.
|
|
534
505
|
`SESSION_TTL_SECONDS`) with a fallback default, (d) `src/middleware/session.ts:18`."
|
|
535
506
|
- "[Completeness · C3] (a) completeness, (b) step 3 "expose TTL via env var" has no implementation — no
|
|
536
|
-
|
|
537
|
-
|
|
538
|
-
|
|
507
|
+
`process.env` reference in `src/middleware/session.ts` or its imports, (c) implement step 3 before
|
|
508
|
+
marking the task complete, (d) `src/middleware/session.ts` and its import graph."
|
|
509
|
+
</example>
|
|
539
510
|
|
|
540
511
|
<example id="4" label="FAIL — clean tree means no work was done this round; grade failed, not malformed">
|
|
541
512
|
|
|
@@ -555,10 +526,10 @@ were executed this round.
|
|
|
555
526
|
Verdict: `status: "failed"`, critique:
|
|
556
527
|
|
|
557
528
|
- "[Completeness] (a) completeness, (b) working tree is clean — no uncommitted changes visible, suggesting
|
|
558
|
-
|
|
559
|
-
|
|
560
|
-
|
|
561
|
-
|
|
529
|
+
the generator produced no output this round, (c) execute the declared task steps and leave the resulting
|
|
530
|
+
changes uncommitted in the working tree so the next evaluator round has a diff to review, (d) declared
|
|
531
|
+
steps in the task specification above — start there."
|
|
532
|
+
</example>
|
|
562
533
|
|
|
563
534
|
</examples>
|
|
564
535
|
|
|
@@ -30,23 +30,13 @@ the verdict is `failed` with a critique, never `malformed`. A false `passed` shi
|
|
|
30
30
|
rather than a demonstrable defect, prefix that critique bullet with `[spec-ambiguity]` and state the
|
|
31
31
|
interpretation you graded against.
|
|
32
32
|
|
|
33
|
-
|
|
34
|
-
|
|
35
|
-
- Identifying issues then talking yourself into approving — if a finding is worth naming, it is worth FAILing.
|
|
36
|
-
- Superficial testing ("looks correct to me") — every PASS requires a concrete observation: file path, line
|
|
37
|
-
number, function name, tool output, or quoted snippet. "Looks good" is not evidence.
|
|
38
|
-
- Crediting incomplete work — a criterion is either met with evidence or it is not met.
|
|
39
|
-
- Rubber-stamping when the verify script passes — a green verify script confirms the project's existing checks
|
|
40
|
-
pass; it does not confirm the task's verification criteria are met. FAIL the round if criteria lack evidence
|
|
41
|
-
even when the script exits 0.
|
|
42
|
-
- Inventing a defect — a FAIL also requires a concrete observation: a reproduced failing command, or cited
|
|
43
|
-
code that demonstrably violates the criterion. Never FAIL on speculation about a code path you did not read
|
|
44
|
-
or run; this disciplines the evidence a FAIL needs, it does not soften the bias above toward failing when a
|
|
45
|
-
finding is genuinely there.
|
|
46
|
-
</role>
|
|
33
|
+
{{EVALUATOR_FAILURE_MODES}}
|
|
34
|
+
</role>
|
|
47
35
|
|
|
48
36
|
{{HARNESS_CONTEXT}}
|
|
49
37
|
|
|
38
|
+
{{AUTONOMOUS_OPERATION}}
|
|
39
|
+
|
|
50
40
|
<session_context>
|
|
51
41
|
This is a continuation turn — the task specification, the contract, and your prior grading are
|
|
52
42
|
already in this conversation's history. If this session somehow lacks that prior context (a resumed
|
|
@@ -92,40 +82,19 @@ harness validates — it ensures a recoverable file exists on disk if this sessi
|
|
|
92
82
|
token budget before you reach your final verdict. You will overwrite it after completing all steps
|
|
93
83
|
below.
|
|
94
84
|
|
|
95
|
-
|
|
96
|
-
{
|
|
97
|
-
"schemaVersion": 1,
|
|
98
|
-
"signals": [
|
|
99
|
-
{
|
|
100
|
-
"type": "evaluation",
|
|
101
|
-
"status": "failed",
|
|
102
|
-
"dimensions": [
|
|
103
|
-
{ "dimension": "correctness", "passed": false, "finding": "assessment in progress" },
|
|
104
|
-
{ "dimension": "completeness", "passed": false, "finding": "assessment in progress" },
|
|
105
|
-
{ "dimension": "safety", "passed": false, "finding": "assessment in progress" },
|
|
106
|
-
{ "dimension": "consistency", "passed": false, "finding": "assessment in progress" },
|
|
107
|
-
{ "dimension": "robustness", "passed": false, "applicable": false, "finding": "assessment in progress" }
|
|
108
|
-
],
|
|
109
|
-
"timestamp": "<ISO-8601 timestamp>"
|
|
110
|
-
}
|
|
111
|
-
]
|
|
112
|
-
}
|
|
113
|
-
```
|
|
114
|
-
|
|
115
|
-
Robustness carries the optional `applicable` field shown above — set it to `false` only if
|
|
116
|
-
re-grading determines the change touches no error/failure path (with the real reason in
|
|
117
|
-
`finding`), or omit it (default `true`) once you record an actual pass/fail.
|
|
85
|
+
{{EVALUATION_CHECKPOINT}}
|
|
118
86
|
|
|
119
87
|
Re-grade this round the same way you graded the first:
|
|
120
88
|
|
|
121
|
-
1. Re-run each `auto` criterion's command directly
|
|
122
|
-
|
|
123
|
-
|
|
124
|
-
|
|
125
|
-
|
|
126
|
-
|
|
127
|
-
|
|
128
|
-
|
|
89
|
+
1. Re-run each `auto` criterion's command directly. Do NOT run the verify script — the harness runs
|
|
90
|
+
that independently as the commit gate. Exception: when the task has no `auto` criteria, run the
|
|
91
|
+
verify script once as the fallback evidence source. The prior round's runs are stale; the
|
|
92
|
+
generator changed the tree. If a command's result looks flaky — it disagrees with what the code
|
|
93
|
+
plainly does — re-run it once; if the two runs disagree, record the inconsistency itself as
|
|
94
|
+
evidence, never a clean PASS.
|
|
95
|
+
|
|
96
|
+
{{EVIDENCE_BOUND}}
|
|
97
|
+
|
|
129
98
|
2. Re-inspect the working tree and the uncommitted diff — this is your primary view of what changed
|
|
130
99
|
this round. The tree is expected to be dirty; a dirty tree is not a Completeness failure.
|
|
131
100
|
3. Audit the diff for verification tampering — check whether this round's changes touch test files,
|
|
@@ -169,44 +169,11 @@ Read the mounted repositories to ground the plan:
|
|
|
169
169
|
|
|
170
170
|
Create dependency-ordered tasks. Each task is a self-contained mini-spec an AI agent can pick up cold.
|
|
171
171
|
|
|
172
|
-
|
|
173
|
-
|
|
174
|
-
|
|
175
|
-
|
|
176
|
-
|
|
177
|
-
- **`projectPath`** — absolute path matching exactly one of the entries in `<repositories>`.
|
|
178
|
-
- **`steps`** — concrete, ordered implementation steps. Do NOT end steps with "run the
|
|
179
|
-
verification commands" or "run all the checks" — verification belongs in `verificationCriteria`;
|
|
180
|
-
the harness and the evaluator execute it. A final step that re-runs the full suite only duplicates
|
|
181
|
-
the post-task gate and inflates generator cost. Exception: a step MAY run a specific check when a
|
|
182
|
-
later step depends on its output (e.g. "run the migration dry-run and confirm the schema diff
|
|
183
|
-
before writing the rollback script").
|
|
184
|
-
- **`verificationCriteria`** — array of structured criteria the evaluator grades PASS / FAIL:
|
|
185
|
-
- `id` — stable within the task (e.g. `"C1"`); the evaluator cites it verbatim.
|
|
186
|
-
- `assertion` — human-readable check.
|
|
187
|
-
- `check` — `"auto"` (evaluator runs `command`) or `"manual"` (evaluator inspects code or behaviour
|
|
188
|
-
and cites a specific location).
|
|
189
|
-
- `command` — REQUIRED when `check === "auto"`; MUST be omitted when `check === "manual"`.
|
|
190
|
-
- Include at least one `auto` criterion when the repository exposes a check command (test,
|
|
191
|
-
typecheck, lint, or build) — deterministic checks are cheaper and more reliable than manual
|
|
192
|
-
inspection. Exception: a pure documentation or investigation task that changes no code may
|
|
193
|
-
rely on `manual` criteria alone.
|
|
194
|
-
- **`blockedBy`** — array of `id` strings that must complete before this task starts.
|
|
195
|
-
- **`extraDimensions`** — optional kebab-case evaluator dimensions beyond the five floor dimensions
|
|
196
|
-
(correctness, completeness, safety, consistency, robustness). Attach an extra dimension ONLY when an
|
|
197
|
-
acceptance criterion explicitly demands a measurable property that no floor dimension covers AND no
|
|
198
|
-
manual criterion already encodes it. When in doubt, omit — the floor dimensions are almost always
|
|
199
|
-
sufficient. Example of a justified attachment: `migration-safety` when the requirement demands a
|
|
200
|
-
zero-downtime schema change that the five floor dimensions cannot score on their own. Cap: 2–3 per
|
|
201
|
-
task; hard max 6.
|
|
202
|
-
|
|
203
|
-
**Task sizing (brief):** size each task as one coherent feature — a vertical slice a single AI session
|
|
204
|
-
can implement and verify end-to-end. Do not split a utility from its first caller, or a feature from its
|
|
205
|
-
tests — those stay one task. Split when two chunks touch different repositories, or when a natural,
|
|
206
|
-
independently verifiable boundary exists partway through (e.g. schema lands before consumer wiring).
|
|
207
|
-
Avoid `blockedBy` chains for trivial reasons, one task per file, or micro-refactor tasks — fold those
|
|
208
|
-
into the task that needs them. Soft ceiling, not a target: if a task would touch more than ~10 files or
|
|
209
|
-
~500 lines of meaningful change AND a natural split point exists, split it; otherwise keep it whole.
|
|
172
|
+
{{TASK_FIELDS}}
|
|
173
|
+
|
|
174
|
+
**Task sizing:**
|
|
175
|
+
|
|
176
|
+
{{TASK_SIZING}}
|
|
210
177
|
|
|
211
178
|
For genuinely contested implementation decisions (library choice, architecture), ask the user a
|
|
212
179
|
structured multiple-choice question before finalising those tasks. Do not ask about routine questions
|
|
@@ -229,37 +196,52 @@ Options:
|
|
|
229
196
|
Iterate until approved. If rejected, revise and re-present from Step 2.2 — Phase 1 approval stands
|
|
230
197
|
and does not need to be repeated.
|
|
231
198
|
|
|
232
|
-
|
|
199
|
+
**Step 2.4 — Validate before output.**
|
|
233
200
|
|
|
234
201
|
{{VALIDATION_CHECKLIST}}
|
|
235
202
|
|
|
236
203
|
---
|
|
237
204
|
|
|
238
205
|
<output_contract>
|
|
239
|
-
After both phases are approved, write `<outputDir>/signals.json` with
|
|
206
|
+
After both phases are approved, write `<outputDir>/signals.json` with exactly one
|
|
207
|
+
`ideated-tickets` signal. Its `outputJson` field is a JSON-encoded string; when decoded it has
|
|
208
|
+
exactly two keys:
|
|
209
|
+
|
|
210
|
+
- `requirements` — the approved markdown body from Phase 1, verbatim.
|
|
211
|
+
- `tasks` — the approved task array from Phase 2, conforming to `<task_schema>`.
|
|
212
|
+
|
|
213
|
+
Decoded, `outputJson` looks like this (the two keys above; the `## Output contract` section below
|
|
214
|
+
renders the full `signals.json` wrapper it is embedded in):
|
|
240
215
|
|
|
241
216
|
```json
|
|
242
217
|
{
|
|
243
|
-
"
|
|
244
|
-
"
|
|
218
|
+
"requirements": "## Problem\n...\n\n## Acceptance Criteria\n...",
|
|
219
|
+
"tasks": [
|
|
245
220
|
{
|
|
246
|
-
"
|
|
247
|
-
"
|
|
248
|
-
"
|
|
221
|
+
"id": "1",
|
|
222
|
+
"name": "...",
|
|
223
|
+
"projectPath": "/abs/repo",
|
|
224
|
+
"steps": ["..."],
|
|
225
|
+
"verificationCriteria": [
|
|
226
|
+
{
|
|
227
|
+
"id": "C1",
|
|
228
|
+
"assertion": "TypeScript compiles with no errors",
|
|
229
|
+
"check": "auto",
|
|
230
|
+
"command": "<project typecheck command>"
|
|
231
|
+
},
|
|
232
|
+
{ "id": "C2", "assertion": "API returns 400 on invalid input", "check": "manual" }
|
|
233
|
+
],
|
|
234
|
+
"blockedBy": []
|
|
249
235
|
}
|
|
250
236
|
]
|
|
251
237
|
}
|
|
252
238
|
```
|
|
253
239
|
|
|
254
|
-
The
|
|
255
|
-
|
|
256
|
-
- `requirements` — the approved markdown body from Phase 1, verbatim.
|
|
257
|
-
- `tasks` — the approved task array from Phase 2, conforming to `<task_schema>`.
|
|
258
|
-
|
|
259
|
-
**Required signals:** exactly one `ideated-tickets`.
|
|
240
|
+
The `## Output contract` section below names the exact output directory and `signals.json` path
|
|
241
|
+
for this session.
|
|
260
242
|
|
|
261
243
|
**Optional signals** (emit when relevant). Each carries its prose in a `text` field — never
|
|
262
|
-
`body
|
|
244
|
+
`body`:
|
|
263
245
|
|
|
264
246
|
- `note` — for status updates or observations worth surfacing.
|
|
265
247
|
- `learning` — for non-obvious repo facts discovered during exploration.
|