rungate 0.6.0 → 0.8.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/hooks/ActionVerificationGate.hook.ts +96 -0
- package/hooks/AgentBriefGuard.hook.ts +38 -0
- package/hooks/AgentVerdictCapture.hook.ts +87 -0
- package/hooks/AutoVerifyGate.hook.ts +79 -0
- package/hooks/BashToolGuard.hook.ts +65 -0
- package/hooks/BashWriteGuard.hook.ts +60 -0
- package/hooks/CommitEnforcement.hook.ts +130 -0
- package/hooks/DocHygiene.hook.ts +148 -0
- package/hooks/GateEnforcement.hook.ts +58 -0
- package/hooks/GateResultProtection.hook.ts +69 -0
- package/hooks/HarnessAFKGate.hook.ts +85 -0
- package/hooks/HookCurrency.hook.ts +99 -0
- package/hooks/IssueCloseGuard.hook.ts +149 -0
- package/hooks/MergeGuard.hook.ts +58 -0
- package/hooks/PostCompact.hook.ts +60 -0
- package/hooks/ProjectStateSync.hook.ts +103 -0
- package/hooks/SkillGuard.hook.ts +69 -0
- package/hooks/SpecConformityTrigger.hook.ts +43 -0
- package/hooks/SpecSCGuard.hook.ts +123 -0
- package/hooks/StaleTTLCleanup.hook.ts +141 -0
- package/hooks/TaskCompleted.hook.ts +51 -0
- package/hooks/TeamInit.hook.ts +85 -0
- package/hooks/TestSuiteGuard.hook.ts +70 -0
- package/hooks/TestSuiteRelease.hook.ts +85 -0
- package/hooks/TopLevelCdGuard.hook.ts +108 -0
- package/hooks/VerifyPhaseLock.hook.ts +84 -0
- package/hooks/WorkflowStateGuard.hook.ts +40 -0
- package/hooks/lib/agent-config.json +9 -0
- package/hooks/lib/agentDetection.ts +201 -0
- package/hooks/lib/comment-template.ts +86 -0
- package/hooks/lib/findWorkflow.ts +101 -0
- package/hooks/lib/parseStdin.ts +37 -0
- package/hooks/lib/paths.ts +111 -0
- package/hooks/lib/utils.ts +338 -0
- package/lib/affected-tests.ts +427 -0
- package/lib/agent-audit.ts +2 -3
- package/lib/agent-worktree.ts +287 -0
- package/lib/agents-md-sections.ts +81 -16
- package/lib/bash-write-guard.ts +373 -0
- package/lib/behavioral-cache.ts +150 -2
- package/lib/branch-cleanup.ts +279 -7
- package/lib/branch-protection.ts +790 -0
- package/lib/brief-ac-alignment.ts +104 -0
- package/lib/brief-references.ts +205 -0
- package/lib/bun-version.ts +93 -14
- package/lib/check-remedy.ts +140 -0
- package/lib/ci-checkout.ts +17 -0
- package/lib/ci-history-depth.ts +159 -0
- package/lib/ci-setup-step.ts +96 -0
- package/lib/claude-code-action.ts +85 -0
- package/lib/claude-tool-vocabulary.ts +188 -0
- package/lib/cli-commands.ts +15 -0
- package/lib/commit-sha.ts +42 -0
- package/lib/commit-subject.ts +90 -0
- package/lib/compliance-report.ts +377 -45
- package/lib/compliance.ts +191 -17
- package/lib/config-loader.ts +39 -1
- package/lib/config-trackable.ts +400 -0
- package/lib/conformity.ts +1087 -100
- package/lib/containment.ts +238 -0
- package/lib/content-hash.ts +24 -0
- package/lib/create-sc.ts +249 -16
- package/lib/dev-liveness.ts +180 -0
- package/lib/discovery-branch-points.ts +432 -0
- package/lib/discovery-stability.ts +481 -0
- package/lib/distribution-currency.ts +236 -0
- package/lib/doc-routing.ts +157 -0
- package/lib/evidence-prevalidator.ts +224 -2
- package/lib/evidence-satisfiability.ts +534 -12
- package/lib/failure-intake.ts +323 -0
- package/lib/fallow-availability.ts +157 -0
- package/lib/fs-write-capabilities.ts +232 -0
- package/lib/gate-enforcement.ts +222 -28
- package/lib/gate-names.ts +72 -0
- package/lib/gate-result-protection.ts +427 -0
- package/lib/gate-run-split.ts +147 -0
- package/lib/generators/agents-md.ts +140 -32
- package/lib/git-isolation.ts +140 -0
- package/lib/github.ts +15 -0
- package/lib/google-auth-action.ts +93 -0
- package/lib/harness-project.ts +48 -0
- package/lib/harness-provenance.ts +255 -0
- package/lib/hook-evidence.ts +235 -0
- package/lib/hook-scopes.ts +386 -0
- package/lib/hook-target-tree.ts +398 -0
- package/lib/hook-trees.ts +447 -0
- package/lib/intake-sources.ts +380 -0
- package/lib/js-top-level-code.ts +78 -10
- package/lib/junit-checks.ts +52 -0
- package/lib/late-env-spawn.ts +331 -0
- package/lib/launch-base.ts +376 -0
- package/lib/line-citation.ts +372 -0
- package/lib/orphan-reaper.ts +468 -0
- package/lib/parity-binding.ts +205 -0
- package/lib/paths.ts +96 -3
- package/lib/per-test-timeouts.ts +219 -0
- package/lib/phase2-decomposition.ts +1 -1
- package/lib/plan-coverage.ts +151 -0
- package/lib/prior-branch.ts +201 -8
- package/lib/process-table.ts +146 -0
- package/lib/project-state-derivation.ts +219 -0
- package/lib/project-state-root.ts +258 -0
- package/lib/published-artifact.ts +669 -0
- package/lib/readdir-sorted.ts +45 -0
- package/lib/review-execution-reading.ts +206 -0
- package/lib/review-tool-agreement.ts +433 -0
- package/lib/rule-promotion.ts +234 -0
- package/lib/rule-retirement.ts +525 -0
- package/lib/run-branch.ts +612 -0
- package/lib/rungate-schema.ts +19 -0
- package/lib/runner-home.ts +274 -0
- package/lib/sc-id-cross-tree.ts +103 -15
- package/lib/sc-ids.ts +121 -13
- package/lib/sc-reservations.ts +289 -0
- package/lib/scaffold/ai-review-findings.ts +409 -0
- package/lib/scaffold/ai-review-step.ts +5406 -64
- package/lib/scaffold/defaults.ts +82 -1
- package/lib/scaffold/managed-workflow.ts +587 -192
- package/lib/scaffold/owned-files.ts +152 -45
- package/lib/scaffold/pre-push-hook.ts +49 -5
- package/lib/scaffold/steps.ts +1836 -275
- package/lib/scaffold/workflow-census.ts +217 -0
- package/lib/scaffold/workflow-drift.ts +217 -0
- package/lib/scaffold/write-steps.ts +4 -0
- package/lib/scan-budget.ts +247 -0
- package/lib/scanner.ts +43 -51
- package/lib/scratch-roots.ts +50 -5
- package/lib/screenshot-discovery.ts +121 -0
- package/lib/screenshot-upload.ts +41 -1
- package/lib/secret-patterns.ts +511 -8
- package/lib/security-verdict.ts +1 -1
- package/lib/shell-path-interpolation.ts +594 -0
- package/lib/ship-mode.ts +219 -0
- package/lib/source-slice.ts +294 -0
- package/lib/stale-cleanup.ts +37 -3
- package/lib/suite-flake.ts +278 -0
- package/lib/suite-measurement.ts +23 -4
- package/lib/suite-run-reading.ts +241 -1
- package/lib/suite-timeout-margin.ts +193 -0
- package/lib/test-case-bodies.ts +883 -0
- package/lib/test-case-deletion.ts +653 -0
- package/lib/test-census.ts +97 -29
- package/lib/test-command-positions.ts +274 -0
- package/lib/test-suite-lock.ts +159 -16
- package/lib/text-dependents.ts +875 -0
- package/lib/tool-probe.ts +31 -9
- package/lib/top-level-cd.ts +393 -0
- package/lib/tree-entries.ts +89 -0
- package/lib/ts-ast.ts +200 -0
- package/lib/walk-test-files.ts +97 -0
- package/lib/workflow-actions.ts +263 -0
- package/lib/workflow-security.ts +29 -0
- package/lib/workflow-taint.ts +732 -0
- package/lib/worktree-cleanup.ts +65 -38
- package/lib/worktree-collect.ts +177 -5
- package/lib/worktree-repo.ts +68 -3
- package/package.json +4 -1
- package/scripts/ai-review-findings.ts +45 -0
- package/scripts/check-installed-hook-currency.ts +825 -0
- package/scripts/create-sc.ts +21 -1
- package/scripts/extract-constraints.ts +107 -7
- package/scripts/generate-code-map.ts +23 -30
- package/scripts/scaffold-project.ts +8 -1
- package/scripts/sync-sc-status.ts +77 -16
|
@@ -0,0 +1,45 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* `readdirSync`, sorted (#827).
|
|
3
|
+
*
|
|
4
|
+
* A LEAF module, in the shape `lib/local-calendar-day.ts` established: one
|
|
5
|
+
* exported binding, no relative imports, so there is exactly one thing to
|
|
6
|
+
* mutate and every call site names the same rule.
|
|
7
|
+
*
|
|
8
|
+
* ## Why this exists
|
|
9
|
+
*
|
|
10
|
+
* Directory enumeration order is the filesystem's, and it is not an order:
|
|
11
|
+
* APFS hands entries back in a different sequence from ext4. The scan
|
|
12
|
+
* renders tables straight from these reads, so three generated files —
|
|
13
|
+
* `.claude/rules/specs-routing.md` and the prompt-routing tables in
|
|
14
|
+
* `.claude/agents/discovery.md` and `marcus.md` — came out as a different
|
|
15
|
+
* PERMUTATION on Linux and on macOS, with byte-identical rows.
|
|
16
|
+
*
|
|
17
|
+
* The consequence is sharper than a cosmetic diff: the repository could be a
|
|
18
|
+
* fixed point of its own generator on the machine its files were last
|
|
19
|
+
* written on and on no other. #827 was measured on a Mac and reported five
|
|
20
|
+
* drifting files; CI found three more that no amount of looking on the
|
|
21
|
+
* author's machine would have shown.
|
|
22
|
+
*
|
|
23
|
+
* ## Why one module rather than a `.sort()` per call site
|
|
24
|
+
*
|
|
25
|
+
* The scan exists TWICE — `lib/scanner.ts` and `lib/scaffold/steps.ts` carry
|
|
26
|
+
* parallel copies, which is the duplication #829 removed for the spec scan
|
|
27
|
+
* and which is still there for the rest. Twenty-four `.sort()` calls are
|
|
28
|
+
* twenty-four things to forget; one binding is one.
|
|
29
|
+
* `test/harness-fixed-point.test.ts` asserts that neither file calls
|
|
30
|
+
* `readdirSync` at all, so a new call site cannot reintroduce the class by
|
|
31
|
+
* reaching for the raw import.
|
|
32
|
+
*
|
|
33
|
+
* Plain `.sort()`: the default lexicographic order over UTF-16 code units is
|
|
34
|
+
* locale-independent, unlike `localeCompare`, and a generator whose output
|
|
35
|
+
* depends on the runner's locale is the same defect in a different coat.
|
|
36
|
+
*/
|
|
37
|
+
import { readdirSync } from "fs";
|
|
38
|
+
|
|
39
|
+
export interface ReaddirSortedOptions {
|
|
40
|
+
recursive?: boolean;
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function readdirSorted(dir: string, opts?: ReaddirSortedOptions): string[] {
|
|
44
|
+
return (opts ? readdirSync(dir, opts) : readdirSync(dir)).map(String).sort();
|
|
45
|
+
}
|
|
@@ -0,0 +1,206 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* What did the review run actually DO? (#732 AC-1, AC-2)
|
|
3
|
+
*
|
|
4
|
+
* Both runs on #732's table concluded **success** having posted nothing. The
|
|
5
|
+
* job had no way to notice: `claude-code-action` sets `conclusion: success`
|
|
6
|
+
* when the CLI exits zero, and a CLI that produced no tool calls and no
|
|
7
|
+
* comments exits zero. `0`, `{}`, `null` and `SKIP` are indistinguishable from
|
|
8
|
+
* success — `.claude/rules/config-access.md` — and so is a review that
|
|
9
|
+
* produced no output. `UNMEASURED` is the only reading that is not.
|
|
10
|
+
*
|
|
11
|
+
* Two facts, deliberately separate, because they can disagree and the
|
|
12
|
+
* disagreement is the interesting case:
|
|
13
|
+
*
|
|
14
|
+
* - **tool calls** — what the agent did, read from the action's
|
|
15
|
+
* `execution_file` (`claude-execution-output.json`, a JSON array of SDK
|
|
16
|
+
* messages; a tool call is a `tool_use` content block on an assistant
|
|
17
|
+
* message). Zero is AC-2: the agent executed nothing, so whatever it
|
|
18
|
+
* concluded it concluded from nothing.
|
|
19
|
+
* - **reviews THIS RUN added** — what GitHub holds after, minus what it held
|
|
20
|
+
* before, counted by the caller against the API. Zero is AC-1: nobody can
|
|
21
|
+
* read this run's review.
|
|
22
|
+
*
|
|
23
|
+
* **A DELTA, and not a total, and that is not a refinement.** The first live
|
|
24
|
+
* dispatch of the fixed workflow posted one review on PR #746. A total would
|
|
25
|
+
* then have read `1` for every subsequent run of that PR forever — including
|
|
26
|
+
* a run that posted nothing, which is the exact state #732 is about. A
|
|
27
|
+
* cumulative count makes the second run of any reviewed PR unfalsifiable, so
|
|
28
|
+
* the before-count is taken before the agent starts and the reading is the
|
|
29
|
+
* difference.
|
|
30
|
+
*
|
|
31
|
+
* These are not two deciders answering one question
|
|
32
|
+
* (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape). An agent
|
|
33
|
+
* that called the posting tool and had the API reject it has a non-zero tool
|
|
34
|
+
* count and a zero review count, and that run is a different defect from one
|
|
35
|
+
* that never tried. Folding them into one boolean would lose exactly that.
|
|
36
|
+
*
|
|
37
|
+
* ## The run's own table could not answer AC-2, which is why this exists
|
|
38
|
+
*
|
|
39
|
+
* #732's "tool calls of any kind: 0" is not readable from either run's log.
|
|
40
|
+
* The action prints `Running Claude Code via SDK (full output hidden for
|
|
41
|
+
* security)` and the JSON result block carries `num_turns`,
|
|
42
|
+
* `permission_denials_count` and `total_cost_usd` and no per-tool record.
|
|
43
|
+
* `num_turns: 13` with 4 denials is in fact evidence the agent WAS attempting
|
|
44
|
+
* tool calls. The execution file is where the answer lives, and nothing was
|
|
45
|
+
* reading it.
|
|
46
|
+
*
|
|
47
|
+
* ## One exit constant
|
|
48
|
+
*
|
|
49
|
+
* `scripts/read-review-execution.ts` routes its only refusal through
|
|
50
|
+
* `REVIEW_UNMEASURED_EXIT`, declared once here, so a test can build a mutant
|
|
51
|
+
* with it set to `0` and watch the refusal stop happening. A later branch that
|
|
52
|
+
* inlines `process.exit(1)` would bypass the mutation and the case would go
|
|
53
|
+
* green having witnessed nothing.
|
|
54
|
+
*/
|
|
55
|
+
|
|
56
|
+
/** The single non-zero exit a review reading refuses with. */
|
|
57
|
+
export const REVIEW_UNMEASURED_EXIT = 1;
|
|
58
|
+
|
|
59
|
+
export type ReviewUnmeasuredReason =
|
|
60
|
+
| "EXECUTION_FILE_ABSENT"
|
|
61
|
+
| "EXECUTION_FILE_UNREADABLE"
|
|
62
|
+
| "NO_TOOL_CALLS"
|
|
63
|
+
| "NO_REVIEW_POSTED"
|
|
64
|
+
| "REVIEW_COUNT_UNAVAILABLE"
|
|
65
|
+
| "DIFF_TRUNCATED";
|
|
66
|
+
|
|
67
|
+
export type ReviewExecutionReading = {
|
|
68
|
+
reading: "MEASURED" | "UNMEASURED";
|
|
69
|
+
/** `null` when the execution file could not be read at all. */
|
|
70
|
+
toolCalls: number | null;
|
|
71
|
+
/** Tool calls by tool name, for the annotation — "which door" needs names. */
|
|
72
|
+
toolCallsByName: Record<string, number>;
|
|
73
|
+
/** Reviews this run ADDED — after minus before. `null` when either
|
|
74
|
+
* count could not be taken. */
|
|
75
|
+
reviewsPosted: number | null;
|
|
76
|
+
reasons: ReviewUnmeasuredReason[];
|
|
77
|
+
/** One sentence per reason, in the same order. */
|
|
78
|
+
detail: string[];
|
|
79
|
+
};
|
|
80
|
+
|
|
81
|
+
export type ReviewExecutionInput = {
|
|
82
|
+
/** The contents of `claude-execution-output.json`, or `null` if absent. */
|
|
83
|
+
executionFileRaw: string | null;
|
|
84
|
+
/** Reviews this run ADDED — the PR's review count after the run minus its
|
|
85
|
+
* count before it, or `null` when either could not be taken. A TOTAL would
|
|
86
|
+
* make the second run of any reviewed PR unfalsifiable. */
|
|
87
|
+
reviewsPosted: number | null;
|
|
88
|
+
/** Whether the diff handed to the agent was cut short of the real one. */
|
|
89
|
+
diffTruncated: boolean;
|
|
90
|
+
};
|
|
91
|
+
|
|
92
|
+
function countToolUses(raw: string): Record<string, number> | null {
|
|
93
|
+
let parsed: unknown;
|
|
94
|
+
try {
|
|
95
|
+
parsed = JSON.parse(raw);
|
|
96
|
+
} catch {
|
|
97
|
+
return null;
|
|
98
|
+
}
|
|
99
|
+
if (!Array.isArray(parsed)) return null;
|
|
100
|
+
const byName: Record<string, number> = {};
|
|
101
|
+
for (const message of parsed) {
|
|
102
|
+
const content = (message as { message?: { content?: unknown } })?.message?.content;
|
|
103
|
+
if (!Array.isArray(content)) continue;
|
|
104
|
+
for (const block of content) {
|
|
105
|
+
const b = block as { type?: unknown; name?: unknown };
|
|
106
|
+
if (b?.type !== "tool_use") continue;
|
|
107
|
+
const name = typeof b.name === "string" ? b.name : "<unnamed>";
|
|
108
|
+
byName[name] = (byName[name] ?? 0) + 1;
|
|
109
|
+
}
|
|
110
|
+
}
|
|
111
|
+
return byName;
|
|
112
|
+
}
|
|
113
|
+
|
|
114
|
+
/**
|
|
115
|
+
* The one decider. `reading` and `reasons` are both projections of the same
|
|
116
|
+
* list computed here and nowhere else, so they cannot disagree.
|
|
117
|
+
*/
|
|
118
|
+
export function reviewExecutionReading(input: ReviewExecutionInput): ReviewExecutionReading {
|
|
119
|
+
const reasons: ReviewUnmeasuredReason[] = [];
|
|
120
|
+
const detail: string[] = [];
|
|
121
|
+
const add = (reason: ReviewUnmeasuredReason, why: string) => {
|
|
122
|
+
reasons.push(reason);
|
|
123
|
+
detail.push(why);
|
|
124
|
+
};
|
|
125
|
+
|
|
126
|
+
let toolCallsByName: Record<string, number> = {};
|
|
127
|
+
let toolCalls: number | null = null;
|
|
128
|
+
|
|
129
|
+
if (input.executionFileRaw === null) {
|
|
130
|
+
add(
|
|
131
|
+
"EXECUTION_FILE_ABSENT",
|
|
132
|
+
"the action produced no execution_file, so nothing can say what the run did",
|
|
133
|
+
);
|
|
134
|
+
} else {
|
|
135
|
+
const byName = countToolUses(input.executionFileRaw);
|
|
136
|
+
if (byName === null) {
|
|
137
|
+
add(
|
|
138
|
+
"EXECUTION_FILE_UNREADABLE",
|
|
139
|
+
"the execution file is not a JSON array of SDK messages — a reading that could not be " +
|
|
140
|
+
"taken is not a reading that passed",
|
|
141
|
+
);
|
|
142
|
+
} else {
|
|
143
|
+
toolCallsByName = byName;
|
|
144
|
+
toolCalls = Object.values(byName).reduce((a, b) => a + b, 0);
|
|
145
|
+
if (toolCalls === 0) {
|
|
146
|
+
add(
|
|
147
|
+
"NO_TOOL_CALLS",
|
|
148
|
+
"the agent made zero tool calls of any kind: it executed nothing, so whatever it " +
|
|
149
|
+
"concluded it concluded from nothing (AC-2)",
|
|
150
|
+
);
|
|
151
|
+
}
|
|
152
|
+
}
|
|
153
|
+
}
|
|
154
|
+
|
|
155
|
+
if (input.reviewsPosted === null) {
|
|
156
|
+
add(
|
|
157
|
+
"REVIEW_COUNT_UNAVAILABLE",
|
|
158
|
+
"the pull request's review count could not be taken before or after the run, so 'this " +
|
|
159
|
+
"run posted a review' is unestablished rather than false",
|
|
160
|
+
);
|
|
161
|
+
} else if (input.reviewsPosted === 0) {
|
|
162
|
+
add(
|
|
163
|
+
"NO_REVIEW_POSTED",
|
|
164
|
+
"this run added no review to the pull request: a review nobody can read is " +
|
|
165
|
+
"indistinguishable from one that found nothing (AC-1)",
|
|
166
|
+
);
|
|
167
|
+
}
|
|
168
|
+
|
|
169
|
+
if (input.diffTruncated) {
|
|
170
|
+
add(
|
|
171
|
+
"DIFF_TRUNCATED",
|
|
172
|
+
"the diff handed to the agent was cut short, so part of the pull request was never looked " +
|
|
173
|
+
"at — a partial review reported as a clean one is the same defect in miniature",
|
|
174
|
+
);
|
|
175
|
+
}
|
|
176
|
+
|
|
177
|
+
return {
|
|
178
|
+
reading: reasons.length === 0 ? "MEASURED" : "UNMEASURED",
|
|
179
|
+
toolCalls,
|
|
180
|
+
toolCallsByName,
|
|
181
|
+
reviewsPosted: input.reviewsPosted,
|
|
182
|
+
reasons,
|
|
183
|
+
detail,
|
|
184
|
+
};
|
|
185
|
+
}
|
|
186
|
+
|
|
187
|
+
/** The annotation body. Named reasons, named counts — a warning that does not
|
|
188
|
+
* say what was missing is a warning nobody can act on. */
|
|
189
|
+
export function reviewReadingReport(r: ReviewExecutionReading): string {
|
|
190
|
+
const calls =
|
|
191
|
+
r.toolCalls === null
|
|
192
|
+
? "tool calls: UNREADABLE"
|
|
193
|
+
: `tool calls: ${r.toolCalls}` +
|
|
194
|
+
(r.toolCalls > 0
|
|
195
|
+
? ` (${Object.entries(r.toolCallsByName)
|
|
196
|
+
.map(([n, c]) => `${n}×${c}`)
|
|
197
|
+
.join(", ")})`
|
|
198
|
+
: "");
|
|
199
|
+
const reviews =
|
|
200
|
+
r.reviewsPosted === null
|
|
201
|
+
? "reviews added by this run: UNCOUNTED"
|
|
202
|
+
: `reviews added by this run: ${r.reviewsPosted}`;
|
|
203
|
+
const head = `${r.reading} — ${calls}; ${reviews}`;
|
|
204
|
+
if (r.reading === "MEASURED") return head;
|
|
205
|
+
return [head, ...r.reasons.map((reason, i) => ` ${reason}: ${r.detail[i]}`)].join("\n");
|
|
206
|
+
}
|
|
@@ -0,0 +1,433 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Does the reviewer's PROMPT agree with the tools the job grants it? (#732)
|
|
3
|
+
*
|
|
4
|
+
* `.github/workflows/claude-review.yml` concluded **success** twice having
|
|
5
|
+
* posted nothing. The issue blames the `SlashCommand` entry on the denylist.
|
|
6
|
+
* That is measured to be wrong — a slash command supplied as the CLI PROMPT is
|
|
7
|
+
* expanded before the model sees it, and `claude -p "/canary" --disallowedTools
|
|
8
|
+
* "SlashCommand,…"` returns the command's output with `permission_denials: []`.
|
|
9
|
+
* Deleting that word would have changed nothing.
|
|
10
|
+
*
|
|
11
|
+
* What was actually blocking it is four independent facts, none of which the
|
|
12
|
+
* issue body names, and every one of which this module reports by its own name:
|
|
13
|
+
*
|
|
14
|
+
* 1. **`Task`.** `/code-review:code-review` says "Use a Haiku agent…", "launch
|
|
15
|
+
* 5 parallel Sonnet agents…" six times, including the step-1 gate whose
|
|
16
|
+
* negative branch says "do not proceed". `Task` is on the denylist, so an
|
|
17
|
+
* agent that cannot run step 1 stops having done nothing.
|
|
18
|
+
* 2. **`Bash`.** Every tool that command's frontmatter declares is `Bash(gh …)`,
|
|
19
|
+
* and its step 6 is "use the gh bash command to comment back". Also denied.
|
|
20
|
+
* 3. **The posting path does not match the allowlist.** The string
|
|
21
|
+
* `create_inline_comment` does not appear in that command at all.
|
|
22
|
+
* 4. **The allowed tool does not exist on this trigger.** Read from the pinned
|
|
23
|
+
* action at the SHA `lib/claude-code-action.ts` declares,
|
|
24
|
+
* `src/mcp/install-mcp-server.ts:152` — the `github_inline_comment` MCP
|
|
25
|
+
* server is installed only when `isEntityContext(context) && context.isPR`,
|
|
26
|
+
* and `src/github/context.ts:54` lists `workflow_dispatch` under
|
|
27
|
+
* `AUTOMATION_EVENT_NAMES`, not `ENTITY_EVENT_NAMES`. This workflow's only
|
|
28
|
+
* trigger is `workflow_dispatch`. So
|
|
29
|
+
* `mcp__github_inline_comment__create_inline_comment` — the single entry on
|
|
30
|
+
* the allowlist, the tool the job exists to call — **cannot exist in any run
|
|
31
|
+
* of it.** A fully working prompt would still have posted nothing.
|
|
32
|
+
*
|
|
33
|
+
* The fourth is the one no amount of reading the prompt finds, and it is why
|
|
34
|
+
* this module takes the TRIGGERS as well as the two tool lists. A grant whose
|
|
35
|
+
* server never starts is indistinguishable from a grant that was never used.
|
|
36
|
+
*
|
|
37
|
+
* ## Why one exported binding and not a file of assertions
|
|
38
|
+
*
|
|
39
|
+
* `.claude/rules/checks-must-be-able-to-fail.md`: one decider, one thing to
|
|
40
|
+
* mutate. Every violation below is reachable from `reviewToolAgreement`, so a
|
|
41
|
+
* mutation of it is visible to every case, and there is no second place that
|
|
42
|
+
* decides what counts.
|
|
43
|
+
*
|
|
44
|
+
* ## Scope, stated rather than implied
|
|
45
|
+
*
|
|
46
|
+
* No YAML parser. This repo ships four runtime dependencies and a guard is not
|
|
47
|
+
* a reason to add a fifth to a published package — the same decision
|
|
48
|
+
* `test/claude-review-no-key.test.ts` records. Everything here is a cut and a
|
|
49
|
+
* regex over the cut, and every cut refuses by name when it cannot find its
|
|
50
|
+
* region, because a slice that missed its section is an empty string and an
|
|
51
|
+
* empty string satisfies every claim of the form "X is not in here".
|
|
52
|
+
*/
|
|
53
|
+
|
|
54
|
+
import { structuralCut } from "./source-slice";
|
|
55
|
+
|
|
56
|
+
/** The names this module reports. One per independent door. */
|
|
57
|
+
export type ReviewToolViolationKind =
|
|
58
|
+
| "PROMPT_IS_A_SLASH_COMMAND"
|
|
59
|
+
| "PROMPT_NAMES_NO_TOOL"
|
|
60
|
+
| "PROMPT_NAMES_A_DENIED_TOOL"
|
|
61
|
+
| "PROMPT_NAMES_AN_UNGRANTED_TOOL"
|
|
62
|
+
| "ALLOWED_TOOL_CANNOT_EXIST_ON_THIS_TRIGGER"
|
|
63
|
+
| "NO_POSTING_TOOL_IS_ALLOWED"
|
|
64
|
+
| "NO_SUBJECT_REACHES_THE_AGENT"
|
|
65
|
+
| "MUTABLE_PLUGIN_MARKETPLACE"
|
|
66
|
+
| "NO_STEP_READS_THE_EXECUTION_OUTPUT";
|
|
67
|
+
|
|
68
|
+
export type ReviewToolViolation = {
|
|
69
|
+
kind: ReviewToolViolationKind;
|
|
70
|
+
detail: string;
|
|
71
|
+
};
|
|
72
|
+
|
|
73
|
+
/**
|
|
74
|
+
* MCP servers the pinned action installs only under an ENTITY event, keyed by
|
|
75
|
+
* the server segment of a tool name. Read from `install-mcp-server.ts` at the
|
|
76
|
+
* pinned SHA; the value is the condition's own words so a reader can go and
|
|
77
|
+
* check it rather than trust this table.
|
|
78
|
+
*
|
|
79
|
+
* A total record rather than a list of strings, so adding a server is a typed
|
|
80
|
+
* edit at one site and the reason travels with the name.
|
|
81
|
+
*/
|
|
82
|
+
export const ENTITY_ONLY_MCP_SERVERS: Record<string, string> = {
|
|
83
|
+
github_inline_comment:
|
|
84
|
+
"install-mcp-server.ts:152 — `isEntityContext(context) && context.isPR && (hasGitHubMcpTools || hasInlineCommentTools)`",
|
|
85
|
+
github_ci:
|
|
86
|
+
"install-mcp-server.ts:177 — `(!isAgentMode || hasGitHubCITools) && isEntityContext(context) && context.isPR && hasWorkflowToken`",
|
|
87
|
+
github_comment:
|
|
88
|
+
"github-comment-server.ts:43 — `update_claude_comment` throws without CLAUDE_COMMENT_ID, which only tag mode sets",
|
|
89
|
+
};
|
|
90
|
+
|
|
91
|
+
/**
|
|
92
|
+
* Events the pinned action treats as AUTOMATION rather than ENTITY —
|
|
93
|
+
* `src/github/context.ts:63` `AUTOMATION_EVENT_NAMES`. A workflow triggered
|
|
94
|
+
* only by these can never satisfy `isEntityContext`.
|
|
95
|
+
*/
|
|
96
|
+
export const AUTOMATION_ONLY_TRIGGERS = [
|
|
97
|
+
"workflow_dispatch",
|
|
98
|
+
"repository_dispatch",
|
|
99
|
+
"schedule",
|
|
100
|
+
"workflow_run",
|
|
101
|
+
] as const;
|
|
102
|
+
|
|
103
|
+
/**
|
|
104
|
+
* Tools that can put a human-readable finding on a pull request. The property
|
|
105
|
+
* AC-4 needs is "a comment naming the defect appears", so this is the set the
|
|
106
|
+
* allowlist must intersect — enumerated, because a server-wide grant beside
|
|
107
|
+
* `pull-requests: write` is the HIGH finding recorded on #732 and widening to
|
|
108
|
+
* `mcp__github` would satisfy a looser check while reintroducing it.
|
|
109
|
+
*/
|
|
110
|
+
export const POSTING_TOOLS = [
|
|
111
|
+
"mcp__github__create_pending_pull_request_review",
|
|
112
|
+
"mcp__github__add_comment_to_pending_review",
|
|
113
|
+
"mcp__github__submit_pending_pull_request_review",
|
|
114
|
+
"mcp__github__create_and_submit_pull_request_review",
|
|
115
|
+
"mcp__github__add_issue_comment",
|
|
116
|
+
"mcp__github_inline_comment__create_inline_comment",
|
|
117
|
+
] as const;
|
|
118
|
+
|
|
119
|
+
/**
|
|
120
|
+
* Tools that let the agent go and FETCH the thing under review. If none is
|
|
121
|
+
* granted, the diff has to arrive in the prompt or the agent is reviewing
|
|
122
|
+
* nothing — which is `NO_SUBJECT_REACHES_THE_AGENT`.
|
|
123
|
+
*/
|
|
124
|
+
export const SUBJECT_FETCHING_TOOLS = [
|
|
125
|
+
"Read",
|
|
126
|
+
"Grep",
|
|
127
|
+
"Glob",
|
|
128
|
+
"Bash",
|
|
129
|
+
"WebFetch",
|
|
130
|
+
"mcp__github__get_pull_request_diff",
|
|
131
|
+
"mcp__github__get_pull_request_files",
|
|
132
|
+
"mcp__github__get_file_contents",
|
|
133
|
+
] as const;
|
|
134
|
+
|
|
135
|
+
/**
|
|
136
|
+
* The step-output name through which the code under review reaches the agent
|
|
137
|
+
* when no fetching tool is granted. A convention, declared here so there is
|
|
138
|
+
* one binding and the rule can say what it is looking for rather than guessing
|
|
139
|
+
* from the shape of an interpolation — `${{ steps.guard.outputs.pr }}` is a
|
|
140
|
+
* step output too, and a PR NUMBER is not the code. That near-miss is the
|
|
141
|
+
* whole reason this is a named output and not "any interpolation".
|
|
142
|
+
*
|
|
143
|
+
* Narrow on purpose, and in the safe direction: an unconventional name costs a
|
|
144
|
+
* refusal somebody fixes by renaming an output. The other direction credits a
|
|
145
|
+
* reviewer that cannot see the diff, which is #732.
|
|
146
|
+
*/
|
|
147
|
+
export const SUBJECT_OUTPUT_NAME = "diff";
|
|
148
|
+
|
|
149
|
+
/** The script whose presence in a step makes that step the reading. */
|
|
150
|
+
export const READING_SCRIPT = "scripts/read-review-execution.ts";
|
|
151
|
+
|
|
152
|
+
export class ReviewSourceUnreadableError extends Error {}
|
|
153
|
+
|
|
154
|
+
/**
|
|
155
|
+
* Every step of the job, as its own string: a `- ` list marker under `steps:`
|
|
156
|
+
* and every line up to the next one.
|
|
157
|
+
*
|
|
158
|
+
* **A comment above a `- ` marker lands in the PRECEDING step, and that is
|
|
159
|
+
* deliberate rather than overlooked.** An earlier draft reattached each
|
|
160
|
+
* trailing comment block to the step it introduces, which reads better and
|
|
161
|
+
* which this file's first measurement showed to be dead: with
|
|
162
|
+
* `withoutComments` below standing between every claim and the prose,
|
|
163
|
+
* restoring the misattribution is **0 of 31 red**, measured. Two deciders
|
|
164
|
+
* answering one question are behaviourally one
|
|
165
|
+
* (`.claude/rules/checks-must-be-able-to-fail.md`, fourth shape), so the
|
|
166
|
+
* cleverer one is gone and `withoutComments` is the single binding.
|
|
167
|
+
*
|
|
168
|
+
* What the misattribution broke before `withoutComments` existed is worth
|
|
169
|
+
* recording, because it is the reason that function is not ceremony: the
|
|
170
|
+
* `setup-bun` step's comment names `READING_SCRIPT` in prose, so a raw-text
|
|
171
|
+
* `filter` returned **two** reading steps for a file with one.
|
|
172
|
+
*/
|
|
173
|
+
export function jobSteps(source: string): string[] {
|
|
174
|
+
const lines = source.split("\n");
|
|
175
|
+
const start = lines.findIndex(l => l.trimStart() === "steps:");
|
|
176
|
+
if (start === -1) throw new ReviewSourceUnreadableError("the job declares no steps:");
|
|
177
|
+
const indentOf = (l: string) => l.length - l.trimStart().length;
|
|
178
|
+
const base = indentOf(lines[start]);
|
|
179
|
+
const steps: string[][] = [];
|
|
180
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
181
|
+
const line = lines[i];
|
|
182
|
+
if (line.trim() !== "" && indentOf(line) <= base) break;
|
|
183
|
+
if (/^\s*- /.test(line)) steps.push([]);
|
|
184
|
+
if (steps.length > 0) steps[steps.length - 1].push(line);
|
|
185
|
+
}
|
|
186
|
+
return steps.map(s => s.join("\n"));
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* A step with its comments removed.
|
|
191
|
+
*
|
|
192
|
+
* A claim about a step's CONFIGURATION must not be satisfiable by a sentence
|
|
193
|
+
* ABOUT that configuration. `lib/review-tool-agreement.ts`'s own first draft
|
|
194
|
+
* asked `/execution_file/.test(source)` of the whole file and was satisfied by
|
|
195
|
+
* a comment in a different step — found by the reviewer this module exists to
|
|
196
|
+
* repair, posted as an inline comment on PR #746.
|
|
197
|
+
*/
|
|
198
|
+
export function withoutComments(step: string): string {
|
|
199
|
+
return step
|
|
200
|
+
.split("\n")
|
|
201
|
+
.filter(l => !l.trimStart().startsWith("#"))
|
|
202
|
+
.join("\n");
|
|
203
|
+
}
|
|
204
|
+
|
|
205
|
+
/**
|
|
206
|
+
* The YAML block an anchor line opens: that line, plus every following line
|
|
207
|
+
* indented deeper than it, stopping at the first line indented the same or
|
|
208
|
+
* less. No length parameter anywhere (#664) — the region ends where the block
|
|
209
|
+
* ends, which is the only boundary that cannot decay as the file grows.
|
|
210
|
+
*
|
|
211
|
+
* Refuses on zero or two occurrences of the anchor, for `lib/source-slice.ts`'s
|
|
212
|
+
* reason: not-found and found-twice are both refusals, and a confident answer
|
|
213
|
+
* about the wrong region is worse than no answer.
|
|
214
|
+
*/
|
|
215
|
+
export function blockUnder(source: string, anchor: string): string {
|
|
216
|
+
const lines = source.split("\n");
|
|
217
|
+
const hits = lines.map((l, i) => [l, i] as const).filter(([l]) => l.trimStart().startsWith(anchor));
|
|
218
|
+
if (hits.length !== 1) {
|
|
219
|
+
throw new ReviewSourceUnreadableError(
|
|
220
|
+
`the anchor ${JSON.stringify(anchor)} opens ${hits.length} lines, not one`,
|
|
221
|
+
);
|
|
222
|
+
}
|
|
223
|
+
const start = hits[0][1];
|
|
224
|
+
const indentOf = (l: string) => l.length - l.trimStart().length;
|
|
225
|
+
const base = indentOf(lines[start]);
|
|
226
|
+
const body = [lines[start]];
|
|
227
|
+
for (let i = start + 1; i < lines.length; i++) {
|
|
228
|
+
if (lines[i].trim() !== "" && indentOf(lines[i]) <= base) break;
|
|
229
|
+
body.push(lines[i]);
|
|
230
|
+
}
|
|
231
|
+
return body.join("\n");
|
|
232
|
+
}
|
|
233
|
+
|
|
234
|
+
/** Every `mcp__server__tool` token in a string, de-duplicated, in order. */
|
|
235
|
+
export function mcpToolsNamedIn(text: string): string[] {
|
|
236
|
+
const seen = new Set<string>();
|
|
237
|
+
for (const m of text.matchAll(/mcp__[a-z0-9_]+__[a-z0-9_]+/gi)) seen.add(m[0]);
|
|
238
|
+
return [...seen];
|
|
239
|
+
}
|
|
240
|
+
|
|
241
|
+
/** The server segment of `mcp__server__tool`, or `null` for a built-in. */
|
|
242
|
+
export function mcpServerOf(tool: string): string | null {
|
|
243
|
+
const m = /^mcp__([a-z0-9_]+)__[a-z0-9_]+$/i.exec(tool);
|
|
244
|
+
return m ? m[1] : null;
|
|
245
|
+
}
|
|
246
|
+
|
|
247
|
+
function quotedListAfter(claudeArgs: string, flag: string): string[] | null {
|
|
248
|
+
const m = new RegExp(`--${flag}\\s+"([^"]*)"`).exec(claudeArgs);
|
|
249
|
+
if (!m) return null;
|
|
250
|
+
return m[1]
|
|
251
|
+
.split(",")
|
|
252
|
+
.map(s => s.trim())
|
|
253
|
+
.filter(Boolean);
|
|
254
|
+
}
|
|
255
|
+
|
|
256
|
+
/**
|
|
257
|
+
* The regions this check is about, cut structurally. Exported so a case can
|
|
258
|
+
* assert the cuts are bounded before asserting anything about their contents —
|
|
259
|
+
* every claim below is of the form "X is not in here", and all of them hold
|
|
260
|
+
* vacuously of an empty string.
|
|
261
|
+
*/
|
|
262
|
+
export function reviewWorkflowRegions(source: string): {
|
|
263
|
+
triggerBlock: string;
|
|
264
|
+
reviewStep: string;
|
|
265
|
+
prompt: string;
|
|
266
|
+
claudeArgs: string;
|
|
267
|
+
} {
|
|
268
|
+
const triggerBlock = structuralCut(source, "\non:", { endAnchor: "\npermissions:" });
|
|
269
|
+
const reviewStep = blockUnder(source, "- name: Review");
|
|
270
|
+
|
|
271
|
+
// The prompt is a YAML block scalar: `prompt: |` and everything indented
|
|
272
|
+
// under it, or a single-line `prompt: "…"`. Cut by indentation rather than
|
|
273
|
+
// by a character count (#664) — the region ends where the block ends.
|
|
274
|
+
const promptBlock = blockUnder(reviewStep, "prompt:");
|
|
275
|
+
const prompt = promptBlock.replace(/^\s*prompt:[ \t]*[|>]?[-+]?[ \t]*/, "").replace(/^"|"$/g, "");
|
|
276
|
+
if (prompt.trim() === "") {
|
|
277
|
+
throw new ReviewSourceUnreadableError("the Review step's prompt is empty");
|
|
278
|
+
}
|
|
279
|
+
|
|
280
|
+
const claudeArgs = blockUnder(reviewStep, "claude_args:");
|
|
281
|
+
|
|
282
|
+
return { triggerBlock, reviewStep, prompt, claudeArgs };
|
|
283
|
+
}
|
|
284
|
+
|
|
285
|
+
/**
|
|
286
|
+
* The one decider. Everything a case asserts about #732 comes back from here.
|
|
287
|
+
*
|
|
288
|
+
* `source` is the whole workflow file; the regions are cut inside, so a caller
|
|
289
|
+
* cannot accidentally ask the question of the wrong slice.
|
|
290
|
+
*/
|
|
291
|
+
export function reviewToolAgreement(source: string): ReviewToolViolation[] {
|
|
292
|
+
const { triggerBlock, reviewStep, prompt, claudeArgs } = reviewWorkflowRegions(source);
|
|
293
|
+
|
|
294
|
+
const allowed = quotedListAfter(claudeArgs, "allowedTools") ?? [];
|
|
295
|
+
const denied = quotedListAfter(claudeArgs, "disallowedTools") ?? [];
|
|
296
|
+
const v: ReviewToolViolation[] = [];
|
|
297
|
+
|
|
298
|
+
// 1. The headline, kept because it is still a defect even though the
|
|
299
|
+
// mechanism the issue gives for it is wrong: a `/plugin:command` prompt
|
|
300
|
+
// makes the review's instructions live in a MUTABLE git ref that nothing
|
|
301
|
+
// in this repository pins or reads.
|
|
302
|
+
if (prompt.trim().startsWith("/")) {
|
|
303
|
+
v.push({
|
|
304
|
+
kind: "PROMPT_IS_A_SLASH_COMMAND",
|
|
305
|
+
detail:
|
|
306
|
+
`the prompt is ${JSON.stringify(prompt.trim().split("\n")[0])} — the instructions it ` +
|
|
307
|
+
`expands to are not in this repository, so nothing here can check which tools they need`,
|
|
308
|
+
});
|
|
309
|
+
}
|
|
310
|
+
|
|
311
|
+
// 2. A prompt that names no tool is the vacuous case: every agreement claim
|
|
312
|
+
// below holds trivially of it, which is exactly the shape that let two
|
|
313
|
+
// runs conclude success.
|
|
314
|
+
const promptTools = mcpToolsNamedIn(prompt);
|
|
315
|
+
if (promptTools.length === 0) {
|
|
316
|
+
v.push({
|
|
317
|
+
kind: "PROMPT_NAMES_NO_TOOL",
|
|
318
|
+
detail: "the prompt names no mcp tool at all, so it cannot be checked against the allowlist",
|
|
319
|
+
});
|
|
320
|
+
}
|
|
321
|
+
|
|
322
|
+
for (const tool of promptTools) {
|
|
323
|
+
if (denied.some(d => tool === d || tool.startsWith(`${d}__`))) {
|
|
324
|
+
v.push({
|
|
325
|
+
kind: "PROMPT_NAMES_A_DENIED_TOOL",
|
|
326
|
+
detail: `the prompt tells the agent to use ${tool}, which --disallowedTools denies`,
|
|
327
|
+
});
|
|
328
|
+
} else if (!allowed.some(a => tool === a || tool.startsWith(`${a}__`))) {
|
|
329
|
+
v.push({
|
|
330
|
+
kind: "PROMPT_NAMES_AN_UNGRANTED_TOOL",
|
|
331
|
+
detail: `the prompt tells the agent to use ${tool}, which --allowedTools does not grant`,
|
|
332
|
+
});
|
|
333
|
+
}
|
|
334
|
+
}
|
|
335
|
+
|
|
336
|
+
// 3. The door nothing in the prompt can show: a grant whose server the
|
|
337
|
+
// pinned action never starts under this workflow's triggers.
|
|
338
|
+
const triggers = [...triggerBlock.matchAll(/^\s{2}([a-z_]+):/gm)].map(m => m[1]);
|
|
339
|
+
const everyTriggerIsAutomation =
|
|
340
|
+
triggers.length > 0 &&
|
|
341
|
+
triggers.every(t => (AUTOMATION_ONLY_TRIGGERS as readonly string[]).includes(t));
|
|
342
|
+
if (everyTriggerIsAutomation) {
|
|
343
|
+
for (const tool of allowed) {
|
|
344
|
+
const server = mcpServerOf(tool);
|
|
345
|
+
const why = server ? ENTITY_ONLY_MCP_SERVERS[server] : undefined;
|
|
346
|
+
if (why) {
|
|
347
|
+
v.push({
|
|
348
|
+
kind: "ALLOWED_TOOL_CANNOT_EXIST_ON_THIS_TRIGGER",
|
|
349
|
+
detail:
|
|
350
|
+
`--allowedTools grants ${tool}, whose MCP server is installed only under an entity ` +
|
|
351
|
+
`event (${why}). This workflow's triggers are ${triggers.join(", ")}, all of which ` +
|
|
352
|
+
`the action classifies as automation events, so the tool never exists`,
|
|
353
|
+
});
|
|
354
|
+
}
|
|
355
|
+
}
|
|
356
|
+
}
|
|
357
|
+
|
|
358
|
+
// 4. Nothing on the allowlist can write a finding anywhere a human will see.
|
|
359
|
+
if (!allowed.some(a => (POSTING_TOOLS as readonly string[]).includes(a))) {
|
|
360
|
+
v.push({
|
|
361
|
+
kind: "NO_POSTING_TOOL_IS_ALLOWED",
|
|
362
|
+
detail:
|
|
363
|
+
`--allowedTools grants [${allowed.join(", ")}], none of which can post a review or a ` +
|
|
364
|
+
`comment. A review that cannot be read is indistinguishable from one that found nothing`,
|
|
365
|
+
});
|
|
366
|
+
}
|
|
367
|
+
|
|
368
|
+
// 5. With no fetching tool granted, the diff has to arrive IN the prompt.
|
|
369
|
+
// Interpolating a prior step's output is the only way it can.
|
|
370
|
+
const canFetch = allowed.some(a => (SUBJECT_FETCHING_TOOLS as readonly string[]).includes(a));
|
|
371
|
+
const promptCarriesSubject = new RegExp(
|
|
372
|
+
`\\$\\{\\{\\s*steps\\.[A-Za-z0-9_-]+\\.outputs\\.${SUBJECT_OUTPUT_NAME}\\s*\\}\\}`,
|
|
373
|
+
).test(prompt);
|
|
374
|
+
if (!canFetch && !promptCarriesSubject) {
|
|
375
|
+
v.push({
|
|
376
|
+
kind: "NO_SUBJECT_REACHES_THE_AGENT",
|
|
377
|
+
detail:
|
|
378
|
+
`no tool on the allowlist can fetch the code and the prompt interpolates no ` +
|
|
379
|
+
`steps.*.outputs.${SUBJECT_OUTPUT_NAME}, so the agent is asked to review code it has ` +
|
|
380
|
+
`no way to see`,
|
|
381
|
+
});
|
|
382
|
+
}
|
|
383
|
+
|
|
384
|
+
// 6. #630 AC-16: plugin code loaded from a ref the publisher can repoint,
|
|
385
|
+
// in a file where every `uses:` is SHA-pinned.
|
|
386
|
+
if (/^\s*plugin_marketplaces\s*:/m.test(reviewStep)) {
|
|
387
|
+
v.push({
|
|
388
|
+
kind: "MUTABLE_PLUGIN_MARKETPLACE",
|
|
389
|
+
detail:
|
|
390
|
+
"the Review step declares plugin_marketplaces, which loads executing code from a MUTABLE " +
|
|
391
|
+
"git ref while every uses: in this file is pinned to a commit SHA (#630 AC-16)",
|
|
392
|
+
});
|
|
393
|
+
}
|
|
394
|
+
|
|
395
|
+
// 7. AC-1/AC-2: something has to read what the run actually did, or a run
|
|
396
|
+
// that made zero tool calls concludes success exactly as before.
|
|
397
|
+
//
|
|
398
|
+
// Asked of the STEP, not the document. The first draft of this check was
|
|
399
|
+
// `/execution_file/.test(source)` and was caught by the reviewer this
|
|
400
|
+
// very change exists to repair: the string already appears in a comment
|
|
401
|
+
// in the Review step, so the check was satisfied by prose about itself —
|
|
402
|
+
// `.claude/rules/checks-must-be-able-to-fail.md` §"search the slice, not
|
|
403
|
+
// the document", which that rule has now caught in its own subject four
|
|
404
|
+
// times. The step is located by the SCRIPT it runs rather than by its
|
|
405
|
+
// `name:`, because a name is prose and the invocation is the binding.
|
|
406
|
+
const readingSteps = jobSteps(source)
|
|
407
|
+
.map(withoutComments)
|
|
408
|
+
.filter(s => s.includes(READING_SCRIPT));
|
|
409
|
+
if (readingSteps.length !== 1) {
|
|
410
|
+
v.push({
|
|
411
|
+
kind: "NO_STEP_READS_THE_EXECUTION_OUTPUT",
|
|
412
|
+
detail:
|
|
413
|
+
`${readingSteps.length} steps run ${READING_SCRIPT}, not one. A run that made zero tool ` +
|
|
414
|
+
`calls and posted nothing then concludes success — which is #732's whole subject`,
|
|
415
|
+
});
|
|
416
|
+
} else {
|
|
417
|
+
const step = readingSteps[0];
|
|
418
|
+
const missing = [
|
|
419
|
+
/execution_file/.test(step) ? null : "it does not read the action's execution_file output",
|
|
420
|
+
/^\s*if:\s*always\(\)\s*$/m.test(step)
|
|
421
|
+
? null
|
|
422
|
+
: "it is not `if: always()`, so the interesting case — the review step reporting success — skips it",
|
|
423
|
+
].filter(Boolean);
|
|
424
|
+
if (missing.length > 0) {
|
|
425
|
+
v.push({
|
|
426
|
+
kind: "NO_STEP_READS_THE_EXECUTION_OUTPUT",
|
|
427
|
+
detail: `the step running ${READING_SCRIPT} cannot do its job: ${missing.join("; ")}`,
|
|
428
|
+
});
|
|
429
|
+
}
|
|
430
|
+
}
|
|
431
|
+
|
|
432
|
+
return v;
|
|
433
|
+
}
|