rungate 0.4.0 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/config/behavioral-sc-map.json +170 -0
- package/config/brief-policies.json +28 -0
- package/config/ceremony-profiles.json +207 -0
- package/config/check-registry.json +114 -0
- package/config/matcher-registry.json +142 -0
- package/config/prove-evidence.schema.json +65 -0
- package/config/spec-policies.json +36 -0
- package/config/universal-rules.yaml +15 -0
- package/config/workflow-schema.json +279 -0
- package/lib/ac-quality.ts +119 -0
- package/lib/cli-commands.ts +7 -1
- package/lib/config-loader.ts +129 -5
- package/lib/conformity.ts +1122 -31
- package/lib/context-docs.ts +182 -0
- package/lib/create-brief.ts +22 -17
- package/lib/create-sc.ts +35 -8
- package/lib/doc-scan-targets.ts +435 -0
- package/lib/gate-check-audit.ts +9 -9
- package/lib/github.ts +195 -0
- package/lib/js-top-level-code.ts +27 -2
- package/lib/organize.ts +129 -40
- package/lib/package-closure.ts +387 -0
- package/lib/project-aliases.ts +100 -0
- package/lib/release-preflight.ts +352 -48
- package/lib/rungate-schema.ts +23 -1
- package/lib/sc-cover.ts +92 -1
- package/lib/sc-ids.ts +151 -0
- package/lib/scaffold/defaults.ts +40 -0
- package/lib/scaffold/managed-workflow.ts +125 -27
- package/lib/scaffold/owned-files.ts +6 -0
- package/lib/scaffold/profiles.ts +94 -0
- package/lib/scaffold/rehearsal.ts +355 -0
- package/lib/scaffold/steps.ts +787 -105
- package/lib/scaffold/write-steps.ts +208 -0
- package/lib/scratch-cleanup.ts +198 -0
- package/lib/scratch-roots.ts +94 -0
- package/lib/source-anchor.ts +144 -0
- package/lib/success-doc.ts +779 -0
- package/lib/suite-run-reading.ts +194 -0
- package/lib/test-census.ts +199 -0
- package/lib/trajectory.ts +377 -0
- package/lib/validators/spec-validators.ts +52 -35
- package/lib/workflow-security.ts +152 -7
- package/lib/workflow-test-harness.ts +349 -0
- package/lib/worktree-cleanup.ts +123 -7
- package/lib/worktree-repo.ts +165 -0
- package/package.json +10 -1
- package/scripts/audit-specs.ts +134 -37
- package/scripts/audit-transcript.ts +259 -0
- package/scripts/create-sc.ts +15 -1
- package/scripts/create-success.ts +162 -0
- package/scripts/detect-sc-drift.ts +200 -0
- package/scripts/extract-constraints.ts +16 -36
- package/scripts/scaffold-project.ts +51 -137
- package/scripts/split-spec.ts +315 -0
- package/scripts/sync-sc-status.ts +751 -0
- package/specs/SPEC-TEMPLATE.md +200 -0
- package/templates/SUCCESS-TEMPLATE.md +72 -0
- package/templates/agent-briefs/_shared.md +9 -0
- package/templates/agent-briefs/aditi.md +46 -0
- package/templates/agent-briefs/discovery.md +37 -0
- package/templates/agent-briefs/marcus.md +39 -0
- package/templates/agent-briefs/quinn.md +67 -0
- package/templates/agent-briefs/rook.md +40 -0
- package/templates/agent-briefs/serena.md +39 -0
package/lib/workflow-security.ts
CHANGED
|
@@ -4,6 +4,56 @@ const SHELL_METACHARACTERS = /[;&|$`\n\r"'\\(){}[\]<>!~*?#]/;
|
|
|
4
4
|
const PATH_TRAVERSAL = /\.\.($|[/\\])/;
|
|
5
5
|
const NULL_BYTE = /\x00/;
|
|
6
6
|
|
|
7
|
+
/**
|
|
8
|
+
* Why a path may not be contributed by an agent, or `null` if it may.
|
|
9
|
+
*
|
|
10
|
+
* ONE table, ONE decider (#400). Before this there were two: a regex set here,
|
|
11
|
+
* checked at staging time, and a segment walk in `workflows/ship.js`'s
|
|
12
|
+
* `escapesWorktree`, checked at collection time. They agreed on traversal and
|
|
13
|
+
* on nothing else, so the audit accepted a reported path carrying a shell
|
|
14
|
+
* metacharacter, a null byte or 500+ characters, declared the run clean, and
|
|
15
|
+
* staging refused it two phases later with a different message.
|
|
16
|
+
*
|
|
17
|
+
* `boundary` separates the two reasons a path can be refused, because they are
|
|
18
|
+
* different things to tell an operator:
|
|
19
|
+
*
|
|
20
|
+
* true — the path does not name a location inside the worktree at all.
|
|
21
|
+
* false — it does, and still cannot be handled safely.
|
|
22
|
+
*
|
|
23
|
+
* "not inside its own worktree" is a wrong sentence to print about
|
|
24
|
+
* `lib/a;rm.ts`, and an operator who reads it goes looking for a path that is
|
|
25
|
+
* not there.
|
|
26
|
+
*
|
|
27
|
+
* The order is the order the reasons are reported in. Absoluteness first
|
|
28
|
+
* because it is the least ambiguous, then the segment walk, then the regexes.
|
|
29
|
+
*
|
|
30
|
+
* The two traversal rules are deliberately both present and deliberately
|
|
31
|
+
* separate. `p.split("/").includes("..")` is the real boundary question; the
|
|
32
|
+
* regex also matches a segment merely ENDING in `..` (`lib/foo..`), which is a
|
|
33
|
+
* legal filename that staging has always refused. Folding it into the boundary
|
|
34
|
+
* rule would label a legal file an escape; dropping it would make the audit
|
|
35
|
+
* permit something staging rejects, which is the whole defect.
|
|
36
|
+
*/
|
|
37
|
+
const CONTRIBUTABLE_PATH_RULES: ReadonlyArray<
|
|
38
|
+
readonly [rule: string, boundary: boolean, fails: (p: string) => boolean]
|
|
39
|
+
> = [
|
|
40
|
+
["an absolute path", true, (p) => p.startsWith("/")],
|
|
41
|
+
["a `..` segment", true, (p) => p.split("/").includes("..")],
|
|
42
|
+
["a segment ending in `..`", false, (p) => PATH_TRAVERSAL.test(p)],
|
|
43
|
+
["a shell metacharacter", false, (p) => SHELL_METACHARACTERS.test(p)],
|
|
44
|
+
["a null byte", false, (p) => NULL_BYTE.test(p)],
|
|
45
|
+
["over 500 characters", false, (p) => p.length > 500],
|
|
46
|
+
];
|
|
47
|
+
|
|
48
|
+
export function pathRefusal(
|
|
49
|
+
p: string,
|
|
50
|
+
): { rule: string; boundary: boolean } | null {
|
|
51
|
+
for (const [rule, boundary, fails] of CONTRIBUTABLE_PATH_RULES) {
|
|
52
|
+
if (fails(p)) return { rule, boundary };
|
|
53
|
+
}
|
|
54
|
+
return null;
|
|
55
|
+
}
|
|
56
|
+
|
|
7
57
|
export function buildSafeGitAdd(filesChanged: string[]): string {
|
|
8
58
|
const validated = validateFilePaths(filesChanged);
|
|
9
59
|
if (validated.rejected.length > 0) {
|
|
@@ -21,13 +71,7 @@ export function validateFilePaths(
|
|
|
21
71
|
const valid: string[] = [];
|
|
22
72
|
const rejected: string[] = [];
|
|
23
73
|
for (const p of paths) {
|
|
24
|
-
if (
|
|
25
|
-
SHELL_METACHARACTERS.test(p) ||
|
|
26
|
-
PATH_TRAVERSAL.test(p) ||
|
|
27
|
-
NULL_BYTE.test(p) ||
|
|
28
|
-
p.startsWith("/") ||
|
|
29
|
-
p.length > 500
|
|
30
|
-
) {
|
|
74
|
+
if (pathRefusal(p)) {
|
|
31
75
|
rejected.push(p);
|
|
32
76
|
} else {
|
|
33
77
|
valid.push(p);
|
|
@@ -64,6 +108,107 @@ export function isSafeBranchName(name: unknown): name is string {
|
|
|
64
108
|
return true;
|
|
65
109
|
}
|
|
66
110
|
|
|
111
|
+
/**
|
|
112
|
+
* Why a repo slug may not reach a shell command, or null if it may (#533).
|
|
113
|
+
*
|
|
114
|
+
* `ship.js` interpolates `repo` and `issueRepo` UNQUOTED into shell text
|
|
115
|
+
* inside agent prompts — six and eleven sites respectively — and checked only
|
|
116
|
+
* that `repo` was present. One function away, the same file validates a
|
|
117
|
+
* branch name before it reaches a `git push`, and `scripts/github-op.ts` has
|
|
118
|
+
* `--title-from-issue` precisely so an issue title never crosses a shell. So
|
|
119
|
+
* the project had already decided that values reaching a shell get checked
|
|
120
|
+
* first, and this is the one that missed.
|
|
121
|
+
*
|
|
122
|
+
* The shape is `parseOwnerRepo`'s in `lib/github.ts`, which is what the
|
|
123
|
+
* GitHub API layer will accept anyway: exactly two segments of
|
|
124
|
+
* `[A-Za-z0-9._-]`, neither of them `.` or `..`. A slug this refuses would
|
|
125
|
+
* have failed at the API boundary regardless — the difference is that it now
|
|
126
|
+
* fails before a shell has seen it, with a sentence naming the argument.
|
|
127
|
+
*
|
|
128
|
+
* REFUSED, never sanitised. Stripping characters out of a repo slug produces
|
|
129
|
+
* a run filed under a different project, which is the defect #114 already
|
|
130
|
+
* cost this repo once.
|
|
131
|
+
*/
|
|
132
|
+
const REPO_SLUG_SEGMENT = /^[A-Za-z0-9._-]+$/;
|
|
133
|
+
|
|
134
|
+
/**
|
|
135
|
+
* The refusals that are about what a slug MEANS rather than which characters
|
|
136
|
+
* it uses — the same shape `isSafeBranchName` takes below, where the regex
|
|
137
|
+
* is followed by git's own rules.
|
|
138
|
+
*
|
|
139
|
+
* A leading `-` is the one that matters and it is the reason that function
|
|
140
|
+
* gives: git and gh both read it as a flag. No GitHub owner or repository
|
|
141
|
+
* name may begin with `-` or `.` or end with `.` anyway, so refusing them
|
|
142
|
+
* costs nothing a real slug needs.
|
|
143
|
+
*/
|
|
144
|
+
function illegalSlugSegment(p: string): boolean {
|
|
145
|
+
if (!REPO_SLUG_SEGMENT.test(p)) return true;
|
|
146
|
+
return p === "." || p === ".." || p.startsWith("-") || p.startsWith(".") || p.endsWith(".");
|
|
147
|
+
}
|
|
148
|
+
|
|
149
|
+
export function repoSlugRefusal(argName: string, value: unknown): string | null {
|
|
150
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
151
|
+
return `${argName} must be a non-empty string of the form "owner/name"; got ${describeArg(value)}`;
|
|
152
|
+
}
|
|
153
|
+
if (value.length > 255) return `${argName} is ${value.length} characters, which is not a repository slug`;
|
|
154
|
+
const parts = value.split("/");
|
|
155
|
+
if (parts.length !== 2 || parts.some(illegalSlugSegment)) {
|
|
156
|
+
return (
|
|
157
|
+
`${argName} must be "owner/name" using only letters, digits, dot, dash and underscore; ` +
|
|
158
|
+
`got ${describeArg(value)} — it is interpolated unquoted into shell commands`
|
|
159
|
+
);
|
|
160
|
+
}
|
|
161
|
+
return null;
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
/**
|
|
165
|
+
* Why a root directory may not reach a shell command, or null if it may.
|
|
166
|
+
*
|
|
167
|
+
* Deliberately NOT `pathRefusal`, and the difference is the first rule in
|
|
168
|
+
* each: that table refuses an absolute path, because it answers "may an agent
|
|
169
|
+
* contribute this file from inside its worktree". This one REQUIRES an
|
|
170
|
+
* absolute path, because it answers "may the run `cd` here". Same word, two
|
|
171
|
+
* questions, opposite answers — folding them together would make one of the
|
|
172
|
+
* two wrong.
|
|
173
|
+
*
|
|
174
|
+
* Whitespace is refused along with the metacharacters, which `pathRefusal`
|
|
175
|
+
* does not do. A space is harmless in a path and fatal in `cd ${ROOT} && …`
|
|
176
|
+
* unquoted, and these values reach seventy-seven such sites. The honest fix
|
|
177
|
+
* is to quote all of them; until then a run on a path with a space in it
|
|
178
|
+
* silently `cd`s somewhere else, and refusing is the reading an operator can
|
|
179
|
+
* act on.
|
|
180
|
+
*/
|
|
181
|
+
export function rootPathRefusal(argName: string, value: unknown): string | null {
|
|
182
|
+
if (typeof value !== "string" || value.length === 0) {
|
|
183
|
+
return `${argName} must be a non-empty absolute path; got ${describeArg(value)}`;
|
|
184
|
+
}
|
|
185
|
+
if (value.length > 500) return `${argName} is ${value.length} characters, which is not a directory path`;
|
|
186
|
+
if (!value.startsWith("/")) return `${argName} must be an absolute path; got ${describeArg(value)}`;
|
|
187
|
+
if (NULL_BYTE.test(value)) return `${argName} contains a null byte`;
|
|
188
|
+
if (value.split("/").includes("..")) return `${argName} contains a \`..\` segment: ${describeArg(value)}`;
|
|
189
|
+
if (/\s/.test(value)) {
|
|
190
|
+
return (
|
|
191
|
+
`${argName} contains whitespace: ${describeArg(value)} — it is interpolated unquoted into ` +
|
|
192
|
+
`shell commands, so the run would cd somewhere other than where you meant`
|
|
193
|
+
);
|
|
194
|
+
}
|
|
195
|
+
if (SHELL_METACHARACTERS.test(value)) {
|
|
196
|
+
return `${argName} contains a shell metacharacter: ${describeArg(value)} — it is interpolated unquoted into shell commands`;
|
|
197
|
+
}
|
|
198
|
+
return null;
|
|
199
|
+
}
|
|
200
|
+
|
|
201
|
+
/**
|
|
202
|
+
* An argument value, quoted and truncated, for a refusal message.
|
|
203
|
+
*
|
|
204
|
+
* Truncated because the value is attacker-shaped by assumption at the point
|
|
205
|
+
* this runs, and a refusal is printed, logged and pasted into issues.
|
|
206
|
+
*/
|
|
207
|
+
function describeArg(value: unknown): string {
|
|
208
|
+
if (typeof value !== "string") return value === undefined ? "nothing" : JSON.stringify(value) ?? String(value);
|
|
209
|
+
return JSON.stringify(value.length > 80 ? value.slice(0, 80) + "…" : value);
|
|
210
|
+
}
|
|
211
|
+
|
|
67
212
|
export function resolveEvidencePath(
|
|
68
213
|
basePath: string,
|
|
69
214
|
evidencePath: string,
|
|
@@ -0,0 +1,349 @@
|
|
|
1
|
+
/**
|
|
2
|
+
* Execute a workflow script with no agents, and record what it did (#61).
|
|
3
|
+
*
|
|
4
|
+
* WHAT THIS IS FOR
|
|
5
|
+
*
|
|
6
|
+
* `workflows/ship.js` is 7,300 lines of orchestration — phase boundaries,
|
|
7
|
+
* ceremony paths, regression targets, schema handoffs — and the only way to
|
|
8
|
+
* find out whether a change to it is correct has been to run the pipeline,
|
|
9
|
+
* which costs real agents and fifteen minutes. So 98 test files read it as
|
|
10
|
+
* TEXT instead, and this repo's own record is that every source-text
|
|
11
|
+
* assertion it has written survived its first mutation.
|
|
12
|
+
*
|
|
13
|
+
* The harness earns its place by doing the one thing none of the other
|
|
14
|
+
* mechanisms can: **executing the whole script and observing the ORDER of
|
|
15
|
+
* what happened.** Not a region lifted out of markers, not a regex over the
|
|
16
|
+
* source — the real control flow, with the real conditionals, reporting the
|
|
17
|
+
* real sequence of phases and agent calls.
|
|
18
|
+
*
|
|
19
|
+
* It runs in well under a second because there is nothing in a workflow
|
|
20
|
+
* script but control flow: the sandbox gives it no filesystem, no subprocess
|
|
21
|
+
* and no imports (#69), so stubbing `agent`, `phase`, `log`, `parallel`,
|
|
22
|
+
* `pipeline` and `workflow` removes everything slow by construction. That is
|
|
23
|
+
* not a happy accident — it is the same property that makes the script
|
|
24
|
+
* untestable by ordinary means.
|
|
25
|
+
*
|
|
26
|
+
* WHAT IT REFUSES TO DO
|
|
27
|
+
*
|
|
28
|
+
* **It will not invent a result for a call site that declares no schema.**
|
|
29
|
+
* A mock that returns `{}` for an unschemaed call is a mock testing itself:
|
|
30
|
+
* the script reads a field, gets `undefined`, and either throws somewhere
|
|
31
|
+
* unrelated or silently takes a branch nobody chose. Those calls are
|
|
32
|
+
* REPORTED, in `unschemaed`, and the caller decides. `0`, `{}` and silence
|
|
33
|
+
* are indistinguishable from success; a named list is not.
|
|
34
|
+
*
|
|
35
|
+
* **It does not check that agent labels are unique.** #61 asked for that and
|
|
36
|
+
* it is false as a requirement: a label legitimately repeats when a
|
|
37
|
+
* remediation round re-runs a call site, which is why
|
|
38
|
+
* `scripts/record-agent-timings.ts` pairs brackets FIFO per label rather
|
|
39
|
+
* than collapsing them. A harness enforcing uniqueness would refuse the
|
|
40
|
+
* runs that matter most.
|
|
41
|
+
*/
|
|
42
|
+
|
|
43
|
+
/** The one exit code every refusal goes through. */
|
|
44
|
+
export const HARNESS_REFUSE_EXIT = 2;
|
|
45
|
+
|
|
46
|
+
export class HarnessRefusal extends Error {}
|
|
47
|
+
|
|
48
|
+
/** The placeholder every synthetic string takes. Named so it is identifiable if it escapes. */
|
|
49
|
+
export const SYNTHETIC_TEXT = "synthetic value produced by the workflow test harness";
|
|
50
|
+
|
|
51
|
+
export interface AgentCall {
|
|
52
|
+
kind: "agent";
|
|
53
|
+
label: string | undefined;
|
|
54
|
+
phase: string | undefined;
|
|
55
|
+
/** Whether the call site declared a schema the harness could synthesise from. */
|
|
56
|
+
schemaed: boolean;
|
|
57
|
+
}
|
|
58
|
+
|
|
59
|
+
export interface PhaseCall {
|
|
60
|
+
kind: "phase";
|
|
61
|
+
title: string;
|
|
62
|
+
}
|
|
63
|
+
|
|
64
|
+
export type HarnessCall = AgentCall | PhaseCall;
|
|
65
|
+
|
|
66
|
+
export interface HarnessRun {
|
|
67
|
+
/** Everything the script did, in the order it did it. */
|
|
68
|
+
calls: HarnessCall[];
|
|
69
|
+
/** Whatever the script returned, or `undefined` if it threw. */
|
|
70
|
+
result: unknown;
|
|
71
|
+
/**
|
|
72
|
+
* The error the script threw, verbatim.
|
|
73
|
+
*
|
|
74
|
+
* NOT swallowed and not converted to a verdict. A harness that reported
|
|
75
|
+
* "the workflow did not complete" would be indistinguishable from one whose
|
|
76
|
+
* own stubs were wrong, which is the failure mode a mock harness is most
|
|
77
|
+
* prone to and least able to see.
|
|
78
|
+
*/
|
|
79
|
+
error: Error | null;
|
|
80
|
+
/** Call sites that declared no schema, by label. Reported, never defaulted. */
|
|
81
|
+
unschemaed: string[];
|
|
82
|
+
/** Wall-clock milliseconds. */
|
|
83
|
+
ms: number;
|
|
84
|
+
}
|
|
85
|
+
|
|
86
|
+
// ── synthetic results ────────────────────────────────────────
|
|
87
|
+
|
|
88
|
+
/**
|
|
89
|
+
* A value conforming to a JSON Schema, built from the schema itself.
|
|
90
|
+
*
|
|
91
|
+
* Generated rather than fixtured, because `agent(prompt, {schema})` hands the
|
|
92
|
+
* schema to the harness at the call site: a per-call-site fixture file would
|
|
93
|
+
* be 39 fixtures that go stale one at a time and silently, and the staleness
|
|
94
|
+
* would show up as the script taking a wrong branch rather than as a failure.
|
|
95
|
+
*
|
|
96
|
+
* Every optional property is populated too, not just the required ones. A
|
|
97
|
+
* minimal object satisfies the schema and exercises only the branches that
|
|
98
|
+
* handle absence, so the harness would systematically run the script's
|
|
99
|
+
* fallback paths and never its main ones.
|
|
100
|
+
*
|
|
101
|
+
* `enum` takes its FIRST value, which is a choice the caller usually needs to
|
|
102
|
+
* override — ceremony tier, gate result, regression target are all enums, and
|
|
103
|
+
* they are the branches worth driving. `overrides` exists for exactly that.
|
|
104
|
+
*/
|
|
105
|
+
export function syntheticFromSchema(schema: any, seed = 1): any {
|
|
106
|
+
if (!schema || typeof schema !== "object") return null;
|
|
107
|
+
if (Array.isArray(schema.enum) && schema.enum.length > 0) return schema.enum[0];
|
|
108
|
+
|
|
109
|
+
switch (schema.type) {
|
|
110
|
+
case "object": {
|
|
111
|
+
const out: Record<string, unknown> = {};
|
|
112
|
+
const props = schema.properties ?? {};
|
|
113
|
+
for (const key of Object.keys(props)) out[key] = syntheticFromSchema(props[key], seed);
|
|
114
|
+
return out;
|
|
115
|
+
}
|
|
116
|
+
case "array":
|
|
117
|
+
// One element, not zero. An empty array satisfies the schema and makes
|
|
118
|
+
// every `.map`, `.length` and `for…of` downstream a no-op, so a harness
|
|
119
|
+
// returning `[]` would report a clean run over a script that never
|
|
120
|
+
// entered a loop.
|
|
121
|
+
return schema.items ? [syntheticFromSchema(schema.items, seed)] : [];
|
|
122
|
+
case "number":
|
|
123
|
+
case "integer":
|
|
124
|
+
return seed;
|
|
125
|
+
case "boolean":
|
|
126
|
+
return true;
|
|
127
|
+
case "string":
|
|
128
|
+
// Several words, not one. The script validates free text in places —
|
|
129
|
+
// an AC statement under five words is refused by the state schema —
|
|
130
|
+
// and a one-word default made every tier stop at the same Discovery
|
|
131
|
+
// refusal, so the harness was measuring its own placeholder rather
|
|
132
|
+
// than the orchestration. Still obviously synthetic, so a value that
|
|
133
|
+
// escapes into a report names itself.
|
|
134
|
+
return SYNTHETIC_TEXT;
|
|
135
|
+
default:
|
|
136
|
+
return schema.properties ? syntheticFromSchema({ ...schema, type: "object" }, seed) : SYNTHETIC_TEXT;
|
|
137
|
+
}
|
|
138
|
+
}
|
|
139
|
+
|
|
140
|
+
// ── loading ──────────────────────────────────────────────────
|
|
141
|
+
|
|
142
|
+
/**
|
|
143
|
+
* A workflow script's executable body, with its `meta` export removed.
|
|
144
|
+
*
|
|
145
|
+
* `export` is a module-level form and the body has to run inside a function,
|
|
146
|
+
* so the declaration cannot come along. It is sliced rather than regexed away
|
|
147
|
+
* because `meta` is required to be a pure literal — the Workflow tool says so
|
|
148
|
+
* — which means the first `\n}\n` after it is its close, and a looser pattern
|
|
149
|
+
* would be a detector narrower than what it detects over a 7,000-line file.
|
|
150
|
+
*/
|
|
151
|
+
export function workflowBody(source: string): string {
|
|
152
|
+
const at = source.indexOf("export const meta");
|
|
153
|
+
if (at < 0) {
|
|
154
|
+
throw new HarnessRefusal(
|
|
155
|
+
"the workflow script has no `export const meta` — either it is not a workflow script, " +
|
|
156
|
+
"or the Workflow tool's own requirement has changed and this harness is reading the wrong thing",
|
|
157
|
+
);
|
|
158
|
+
}
|
|
159
|
+
const end = source.indexOf("\n}\n", at);
|
|
160
|
+
if (end < 0) throw new HarnessRefusal("could not find the end of the workflow's `meta` literal");
|
|
161
|
+
return source.slice(end + 3);
|
|
162
|
+
}
|
|
163
|
+
|
|
164
|
+
// ── running ──────────────────────────────────────────────────
|
|
165
|
+
|
|
166
|
+
export interface RunOptions {
|
|
167
|
+
/** `args` as the script sees it. */
|
|
168
|
+
args?: Record<string, unknown>;
|
|
169
|
+
/**
|
|
170
|
+
* Replace the synthetic result for a given label.
|
|
171
|
+
*
|
|
172
|
+
* A function receives the synthetic value and returns the replacement, so a
|
|
173
|
+
* caller can override one field of a large schema without restating it —
|
|
174
|
+
* restating it is how an override goes stale against a schema change and
|
|
175
|
+
* starts driving a branch nobody meant.
|
|
176
|
+
*/
|
|
177
|
+
overrides?: Record<string, unknown | ((synthetic: any) => unknown)>;
|
|
178
|
+
/**
|
|
179
|
+
* Hard cap on agent calls. A runaway loop in the script under test must
|
|
180
|
+
* fail the harness, not hang the suite — and the cap is a REFUSAL with a
|
|
181
|
+
* count, not a quiet stop, because a quiet stop looks like a short run.
|
|
182
|
+
*/
|
|
183
|
+
maxCalls?: number;
|
|
184
|
+
}
|
|
185
|
+
|
|
186
|
+
export async function runWorkflow(source: string, opts: RunOptions = {}): Promise<HarnessRun> {
|
|
187
|
+
const calls: HarnessCall[] = [];
|
|
188
|
+
const unschemaed: string[] = [];
|
|
189
|
+
const maxCalls = opts.maxCalls ?? 500;
|
|
190
|
+
const overrides = opts.overrides ?? {};
|
|
191
|
+
|
|
192
|
+
const agent = async (_prompt: string, o: any = {}) => {
|
|
193
|
+
const label = o?.label;
|
|
194
|
+
calls.push({ kind: "agent", label, phase: o?.phase, schemaed: !!o?.schema });
|
|
195
|
+
if (calls.filter((c) => c.kind === "agent").length > maxCalls) {
|
|
196
|
+
throw new HarnessRefusal(
|
|
197
|
+
`the workflow made more than ${maxCalls} agent calls — it is looping, and a harness ` +
|
|
198
|
+
`that stopped quietly here would report a short run rather than a runaway one`,
|
|
199
|
+
);
|
|
200
|
+
}
|
|
201
|
+
if (!o?.schema) {
|
|
202
|
+
if (label && !unschemaed.includes(label)) unschemaed.push(label);
|
|
203
|
+
// A string, which is what a schemaless `agent()` really returns. Not
|
|
204
|
+
// `{}`: an object that answers every property access with `undefined`
|
|
205
|
+
// lets the script take branches nobody chose, and the harness would be
|
|
206
|
+
// reporting on its own stub.
|
|
207
|
+
return SYNTHETIC_TEXT;
|
|
208
|
+
}
|
|
209
|
+
const synthetic = syntheticFromSchema(o.schema);
|
|
210
|
+
if (label && label in overrides) {
|
|
211
|
+
const o2 = overrides[label];
|
|
212
|
+
return typeof o2 === "function" ? (o2 as (s: any) => unknown)(synthetic) : o2;
|
|
213
|
+
}
|
|
214
|
+
return synthetic;
|
|
215
|
+
};
|
|
216
|
+
|
|
217
|
+
const globals: Record<string, unknown> = {
|
|
218
|
+
agent,
|
|
219
|
+
phase: (title: string) => calls.push({ kind: "phase", title }),
|
|
220
|
+
log: () => {},
|
|
221
|
+
parallel: async (thunks: Array<() => Promise<unknown>>) =>
|
|
222
|
+
Promise.all((thunks ?? []).map((t) => t().catch(() => null))),
|
|
223
|
+
pipeline: async (items: unknown[], ...stages: Array<(...a: any[]) => Promise<unknown>>) => {
|
|
224
|
+
const out: unknown[] = [];
|
|
225
|
+
for (const [i, item] of (items ?? []).entries()) {
|
|
226
|
+
let v: unknown = item;
|
|
227
|
+
for (const stage of stages) v = await stage(v, item, i);
|
|
228
|
+
out.push(v);
|
|
229
|
+
}
|
|
230
|
+
return out;
|
|
231
|
+
},
|
|
232
|
+
workflow: async () => ({}),
|
|
233
|
+
args: opts.args ?? {},
|
|
234
|
+
budget: { total: null, spent: () => 0, remaining: () => Infinity },
|
|
235
|
+
};
|
|
236
|
+
|
|
237
|
+
const names = Object.keys(globals);
|
|
238
|
+
const started = Date.now();
|
|
239
|
+
let result: unknown;
|
|
240
|
+
let error: Error | null = null;
|
|
241
|
+
try {
|
|
242
|
+
const fn = new Function(...names, `return (async () => {\n${workflowBody(source)}\n})()`);
|
|
243
|
+
result = await fn(...names.map((n) => globals[n]));
|
|
244
|
+
} catch (err) {
|
|
245
|
+
error = err instanceof Error ? err : new Error(String(err));
|
|
246
|
+
}
|
|
247
|
+
return { calls, result, error, unschemaed, ms: Date.now() - started };
|
|
248
|
+
}
|
|
249
|
+
|
|
250
|
+
// ── contracts over a recorded run ────────────────────────────
|
|
251
|
+
|
|
252
|
+
/** The phases `workflows/ship.js` declares, in the order it declares them. */
|
|
253
|
+
export const SHIP_PHASES = [
|
|
254
|
+
"Goal", "Discovery", "Scope", "Implement", "Validate", "Commit", "Verify", "Ship", "Prove",
|
|
255
|
+
] as const;
|
|
256
|
+
|
|
257
|
+
/**
|
|
258
|
+
* Phases that ran out of order, as violations.
|
|
259
|
+
*
|
|
260
|
+
* A subsequence check, not an equality check: a run that stops early, or
|
|
261
|
+
* skips a phase its tier does not need, is correct. A run that goes BACKWARDS
|
|
262
|
+
* is not — `Implement` before `Scope` is the boundary violation #61 names,
|
|
263
|
+
* and it is the one a reordering edit introduces silently.
|
|
264
|
+
*
|
|
265
|
+
* A phase the order does not know is a violation too. Adding a phase to
|
|
266
|
+
* ship.js and not to this list means the harness has stopped describing the
|
|
267
|
+
* script, and defaulting it to "fine" is how a contract check goes blind.
|
|
268
|
+
*/
|
|
269
|
+
export function phaseOrderViolations(calls: HarnessCall[], order: readonly string[] = SHIP_PHASES): string[] {
|
|
270
|
+
const violations: string[] = [];
|
|
271
|
+
let high = -1;
|
|
272
|
+
let highName = "<start>";
|
|
273
|
+
for (const call of calls) {
|
|
274
|
+
if (call.kind !== "phase") continue;
|
|
275
|
+
const at = order.indexOf(call.title);
|
|
276
|
+
if (at < 0) {
|
|
277
|
+
violations.push(
|
|
278
|
+
`phase('${call.title}') is not in the declared phase order — either the order is stale ` +
|
|
279
|
+
`or the phase is new, and a harness that ignored it would stop describing the script`,
|
|
280
|
+
);
|
|
281
|
+
continue;
|
|
282
|
+
}
|
|
283
|
+
// Equal is fine: a phase can be re-entered on a regression path, and
|
|
284
|
+
// re-entry is not a backwards jump.
|
|
285
|
+
if (at < high) violations.push(`phase('${call.title}') ran after phase('${highName}') — the declared order is ${order.join(" → ")}`);
|
|
286
|
+
else {
|
|
287
|
+
high = at;
|
|
288
|
+
highName = call.title;
|
|
289
|
+
}
|
|
290
|
+
}
|
|
291
|
+
return violations;
|
|
292
|
+
}
|
|
293
|
+
|
|
294
|
+
/**
|
|
295
|
+
* Agent calls that ran AFTER the phase they belong to had passed.
|
|
296
|
+
*
|
|
297
|
+
* The other half of a boundary, and narrower than the obvious version. The
|
|
298
|
+
* first draft flagged any agent whose declared phase differed from the
|
|
299
|
+
* current one, and the harness's very first run reported
|
|
300
|
+
* `preload-contexts` — which declares `Discovery` and executes while the run
|
|
301
|
+
* is still in `Goal`, 264 lines before `phase('Discovery')`.
|
|
302
|
+
*
|
|
303
|
+
* That is not a defect. `opts.phase` assigns a PROGRESS GROUP, and the
|
|
304
|
+
* workflow API documents declaring one ahead of the global `phase()` state
|
|
305
|
+
* precisely so parallel stages do not race it. Grouping a preload under the
|
|
306
|
+
* phase it prepares for is the intended usage, and a check that refused it
|
|
307
|
+
* would have been a detector wider than what it detects — loud, wrong, and
|
|
308
|
+
* switched off within a week.
|
|
309
|
+
*
|
|
310
|
+
* Declaring BACKWARDS is the suspicious direction: a call site labelled for
|
|
311
|
+
* `Goal` executing during `Ship` is work that has escaped its phase, and
|
|
312
|
+
* nothing else in the repo can see it, because `phase()` was never called
|
|
313
|
+
* wrongly — the AGENT was placed wrongly.
|
|
314
|
+
*/
|
|
315
|
+
export function agentPhaseViolations(calls: HarnessCall[], order: readonly string[] = SHIP_PHASES): string[] {
|
|
316
|
+
const violations: string[] = [];
|
|
317
|
+
let current: string | null = null;
|
|
318
|
+
for (const call of calls) {
|
|
319
|
+
if (call.kind === "phase") {
|
|
320
|
+
current = call.title;
|
|
321
|
+
continue;
|
|
322
|
+
}
|
|
323
|
+
if (!call.phase || call.phase === current || current === null) continue;
|
|
324
|
+
const declared = order.indexOf(call.phase);
|
|
325
|
+
const now = order.indexOf(current);
|
|
326
|
+
// An unknown phase on either side is `phaseOrderViolations`'s finding,
|
|
327
|
+
// not this one. Reporting it twice would be two deciders on one fact.
|
|
328
|
+
if (declared < 0 || now < 0 || declared >= now) continue;
|
|
329
|
+
violations.push(
|
|
330
|
+
`agent '${call.label ?? "<unlabelled>"}' belongs to phase '${call.phase}' but ran during ` +
|
|
331
|
+
`'${current}', which is later — the call site has escaped its phase`,
|
|
332
|
+
);
|
|
333
|
+
}
|
|
334
|
+
return violations;
|
|
335
|
+
}
|
|
336
|
+
|
|
337
|
+
/**
|
|
338
|
+
* The run's call graph, as a comparable string.
|
|
339
|
+
*
|
|
340
|
+
* This is the pass^k instrument: k runs with the same inputs must produce the
|
|
341
|
+
* same string. Labels and phases only — timings and prompt text are not part
|
|
342
|
+
* of the contract and would make every run differ for reasons nobody cares
|
|
343
|
+
* about, which is how a determinism metric gets switched off.
|
|
344
|
+
*/
|
|
345
|
+
export function callGraph(calls: HarnessCall[]): string {
|
|
346
|
+
return calls
|
|
347
|
+
.map((c) => (c.kind === "phase" ? `phase:${c.title}` : `agent:${c.label ?? "?"}@${c.phase ?? "?"}`))
|
|
348
|
+
.join("\n");
|
|
349
|
+
}
|