unitbob 0.7.8 → 0.7.13
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +88 -80
- package/dist/cli.js +55 -5
- package/dist/files/behavioral.js +72 -5
- package/dist/files/featureStart.js +65 -0
- package/dist/files/features.js +156 -0
- package/dist/files/mapBuild.js +2 -1
- package/dist/files/suiteBuild.js +9 -2
- package/dist/runner/bdd.js +36 -10
- package/dist/runner/failureDigest.js +59 -0
- package/dist/runner/gitRevision.js +20 -0
- package/dist/runner/manifest.js +3 -1
- package/dist/runner/outputTail.js +13 -0
- package/dist/runner/pytestBddPlugin.js +15 -0
- package/dist/verbs/contractPrompt.js +23 -4
- package/dist/verbs/featurePrepare.js +49 -0
- package/dist/verbs/fixPrepare.js +21 -9
- package/dist/verbs/knowledgePrepare.js +39 -0
- package/dist/verbs/mapPrepare.js +29 -3
- package/dist/verbs/putFeature.js +21 -0
- package/dist/verbs/putKnowledge.js +21 -0
- package/dist/verbs/putTests.js +133 -0
- package/dist/verbs/run.js +72 -30
- package/dist/verbs/runLocal.js +56 -14
- package/dist/verbs/suitePrepare.js +3 -1
- package/dist/verbs/suiteReviewPrepare.js +45 -29
- package/dist/verbs/testsPrepare.js +77 -0
- package/dist/verbs/testsReviewPrepare.js +40 -0
- package/dist/wire.js +134 -5
- package/package.json +1 -1
- package/plugin/codex/agents/suite-repair-worker.toml +1 -1
- package/plugin/codex/agents/suite-reviewer.toml +18 -0
|
@@ -0,0 +1,133 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { knowledgePath, parseFeatureId, readKnowledge, readTestsOutput, readTestsRequest, readTestsReviewOutput, testsReviewOutputPath, } from "../files/features.js";
|
|
3
|
+
import { suiteCandidateDigest } from "../files/suiteBuild.js";
|
|
4
|
+
import { enterUrl } from "../links.js";
|
|
5
|
+
import { runBddSuite } from "../runner/bdd.js";
|
|
6
|
+
import { boundReport } from "../runner/boundReport.js";
|
|
7
|
+
import { scenarioTally } from "../runner/failureDigest.js";
|
|
8
|
+
import { gitRevision } from "../runner/gitRevision.js";
|
|
9
|
+
import { placeAdvice } from "../runner/placeAdvice.js";
|
|
10
|
+
import { placeProblem } from "../runner/place.js";
|
|
11
|
+
import { runnerEnvironmentPlaceProblem } from "../runner/placeEnvironment.js";
|
|
12
|
+
import { outputTail } from "../runner/outputTail.js";
|
|
13
|
+
import { Wire } from "../wire.js";
|
|
14
|
+
const OUTPUT_TAIL_CHARS = 2000;
|
|
15
|
+
// The digest of `knowledge.md` on disk against the one the request was written
|
|
16
|
+
// from — both named when they differ, so the reader sees which side moved.
|
|
17
|
+
export function assertKnowledgeUnchanged(projectRoot, featureId, expected) {
|
|
18
|
+
const got = createHash('sha256').update(readKnowledge(projectRoot, featureId)).digest('hex');
|
|
19
|
+
if (got === expected)
|
|
20
|
+
return;
|
|
21
|
+
throw new Error(`${knowledgePath(projectRoot, featureId)}: knowledge.md on disk differs from what the server has — ` +
|
|
22
|
+
`run put-knowledge first, or restore the file, then run this again.\nexpected: ${expected}\n got: ${got}`);
|
|
23
|
+
}
|
|
24
|
+
// `put-tests <feature_id>` (spec 52-3, AC 3.5). The answer is read with its
|
|
25
|
+
// files from disk; the candidate digest is the one the review uses; then the
|
|
26
|
+
// connector runs the feature's tag itself — a report the host already has may
|
|
27
|
+
// be from earlier and over other files, and the proof has to be over these
|
|
28
|
+
// bytes. A runner that could not start or produced no report sends nothing:
|
|
29
|
+
// what it said is printed, and the exit code is non-zero. A 409 or 422 comes
|
|
30
|
+
// back as a WireError already worded by the server, one line per side, and is
|
|
31
|
+
// let through as it is.
|
|
32
|
+
//
|
|
33
|
+
// What the run proves decides what travels with it (spec 52-4, AC 1.10). All
|
|
34
|
+
// red: the run itself as `red_run`, revision and all, the way the known-defect
|
|
35
|
+
// probe sends its own. All green with the reviewer's file beside the answer:
|
|
36
|
+
// that file as `bdd_quality_review`, bound to this candidate. Anything else:
|
|
37
|
+
// no proof — the harness was saved mid-build. A review over a run that is not
|
|
38
|
+
// all green is not sent at all; a review of an older candidate is ignored out
|
|
39
|
+
// loud. And once the server has taken the version, the same run is filed
|
|
40
|
+
// under its digest, so the feature's page shows "N of M checks pass" on it at
|
|
41
|
+
// once rather than on the version before.
|
|
42
|
+
export async function putTests(config, args = [], deps) {
|
|
43
|
+
const wire = new Wire(config);
|
|
44
|
+
const d = {
|
|
45
|
+
runBehavioral: runBddSuite,
|
|
46
|
+
gitRevision,
|
|
47
|
+
putFeatureSuite: (id, upload) => wire.putFeatureSuite(id, upload),
|
|
48
|
+
postRunsBatch: (runs) => wire.postRunsBatch(runs),
|
|
49
|
+
stdout: process.stdout,
|
|
50
|
+
...deps,
|
|
51
|
+
};
|
|
52
|
+
const featureId = parseFeatureId(args[0], 'put-tests');
|
|
53
|
+
const request = readTestsRequest(config.projectRoot, featureId);
|
|
54
|
+
const output = readTestsOutput(config.projectRoot, featureId);
|
|
55
|
+
// The same check `tests-prepare` made, made again here: the file may have
|
|
56
|
+
// been edited in between, and the checks are sealed to the text the server
|
|
57
|
+
// has, not to the text on disk (spec 52-3, edge cases).
|
|
58
|
+
assertKnowledgeUnchanged(config.projectRoot, featureId, request.knowledge_digest);
|
|
59
|
+
const unusable = placeProblem(config.projectRoot) ?? runnerEnvironmentPlaceProblem(config.projectRoot);
|
|
60
|
+
if (unusable)
|
|
61
|
+
throw new Error(unusable);
|
|
62
|
+
// The review file, before the run: one the connector cannot read is a
|
|
63
|
+
// stop, said in one line, and a run before it would be a run for nothing.
|
|
64
|
+
const candidateDigest = suiteCandidateDigest({
|
|
65
|
+
suite_kind: 'behavioral', suite_file: output.suite_file, runner_manifest: output.runner_manifest,
|
|
66
|
+
});
|
|
67
|
+
let review;
|
|
68
|
+
try {
|
|
69
|
+
review = readTestsReviewOutput(config.projectRoot, featureId);
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
d.stdout.write(`${err.message}\n`);
|
|
73
|
+
d.stdout.write('Nothing was sent.\n');
|
|
74
|
+
return 1;
|
|
75
|
+
}
|
|
76
|
+
if (review && review.candidate_digest !== candidateDigest) {
|
|
77
|
+
d.stdout.write(`${testsReviewOutputPath(config.projectRoot, featureId)} is for an older candidate — ignored.\n`);
|
|
78
|
+
review = null;
|
|
79
|
+
}
|
|
80
|
+
let result;
|
|
81
|
+
try {
|
|
82
|
+
result = await d.runBehavioral(config.projectRoot, request.runner, request.feature_path, { only: request.feature_tag });
|
|
83
|
+
}
|
|
84
|
+
catch (err) {
|
|
85
|
+
const advice = placeAdvice(config.projectRoot);
|
|
86
|
+
d.stdout.write(`The runner could not start: ${err.message}\n${advice ? `\n${advice}\n` : ''}`);
|
|
87
|
+
d.stdout.write('Nothing was sent.\n');
|
|
88
|
+
return 1;
|
|
89
|
+
}
|
|
90
|
+
const report = boundReport(request.runner, result);
|
|
91
|
+
if (report === null) {
|
|
92
|
+
d.stdout.write(`The run produced no machine-readable report at ${result.resultPath} (exit code ${result.code}) — ` +
|
|
93
|
+
'it died before the first scenario rather than failing them.\n');
|
|
94
|
+
const tail = outputTail(result, OUTPUT_TAIL_CHARS);
|
|
95
|
+
if (tail)
|
|
96
|
+
d.stdout.write(`${tail}\n`);
|
|
97
|
+
d.stdout.write('Nothing was sent.\n');
|
|
98
|
+
return 1;
|
|
99
|
+
}
|
|
100
|
+
const tally = scenarioTally(request.runner, result.report);
|
|
101
|
+
const allRed = tally !== null && tally.passed === 0 && tally.failed > 0;
|
|
102
|
+
const allGreen = tally !== null && tally.failed === 0 && tally.passed > 0;
|
|
103
|
+
if (review && !allGreen) {
|
|
104
|
+
d.stdout.write(`Review the checks when they all pass — ${stillFailing(tally)}.\n`);
|
|
105
|
+
return 1;
|
|
106
|
+
}
|
|
107
|
+
const proof = review
|
|
108
|
+
? { bdd_quality_review: { ...review.bdd_quality_review, candidate_digest: candidateDigest } }
|
|
109
|
+
: allRed
|
|
110
|
+
? { red_run: { candidate_digest: candidateDigest, revision: d.gitRevision(config.projectRoot), run_result: report } }
|
|
111
|
+
: {};
|
|
112
|
+
const recorded = await d.putFeatureSuite(featureId, {
|
|
113
|
+
suite_file: output.suite_file,
|
|
114
|
+
runner_manifest: output.runner_manifest,
|
|
115
|
+
test_metadata: { ...output.test_metadata, ...proof },
|
|
116
|
+
knowledge_digest: request.knowledge_digest,
|
|
117
|
+
});
|
|
118
|
+
// Said before the run is filed: the upload is done whatever happens next,
|
|
119
|
+
// and a filing that fails leaves the run to the next `check`.
|
|
120
|
+
d.stdout.write(`${recorded.message}\n`);
|
|
121
|
+
const { results } = await d.postRunsBatch([{ suite_digest: recorded.suite_digest, run_result: report }]);
|
|
122
|
+
for (const item of results)
|
|
123
|
+
d.stdout.write(`${item.summary}\n`);
|
|
124
|
+
d.stdout.write(`${enterUrl(config, recorded.url)}\n`);
|
|
125
|
+
return 0;
|
|
126
|
+
}
|
|
127
|
+
// "2 of 5 still fail" — or, for a report the connector could not count, only
|
|
128
|
+
// that it could not.
|
|
129
|
+
function stillFailing(tally) {
|
|
130
|
+
if (tally === null)
|
|
131
|
+
return 'the run’s report could not be read scenario by scenario';
|
|
132
|
+
return `${tally.failed} of ${tally.passed + tally.failed} still fail`;
|
|
133
|
+
}
|
package/dist/verbs/run.js
CHANGED
|
@@ -1,5 +1,5 @@
|
|
|
1
1
|
import { materializeGuardrails } from "../files/guardrails.js";
|
|
2
|
-
import {
|
|
2
|
+
import { FeatureFilesChangedError, materializeBehavioralUnion } from "../files/behavioral.js";
|
|
3
3
|
import { placeProblem } from "../runner/place.js";
|
|
4
4
|
import { runnerEnvironmentPlaceProblem } from "../runner/placeEnvironment.js";
|
|
5
5
|
import { validateStack } from "../runner/precheck.js";
|
|
@@ -7,6 +7,7 @@ import { runRspecSuite } from "../runner/rspec.js";
|
|
|
7
7
|
import { runVitestSuite, testPathsOf } from "../runner/vitest.js";
|
|
8
8
|
import { runPytestSuite } from "../runner/pytest.js";
|
|
9
9
|
import { runBddSuite } from "../runner/bdd.js";
|
|
10
|
+
import { outputTail } from "../runner/outputTail.js";
|
|
10
11
|
import { enterUrl } from "../links.js";
|
|
11
12
|
import { boundReport } from "../runner/boundReport.js";
|
|
12
13
|
import { Wire } from "../wire.js";
|
|
@@ -27,7 +28,7 @@ export async function runOnly(config, digests, deps) {
|
|
|
27
28
|
function resolve(config, deps) {
|
|
28
29
|
const wire = new Wire(config);
|
|
29
30
|
return {
|
|
30
|
-
|
|
31
|
+
getSuiteIndex: () => wire.getSuiteIndex(),
|
|
31
32
|
postRunsBatch: (runs) => wire.postRunsBatch(runs),
|
|
32
33
|
// The whole envelope, support files and all: a branch is a set of files
|
|
33
34
|
// since spec one-place-per-rule, §6, and picking `path` and `content` out of it here was
|
|
@@ -37,7 +38,9 @@ function resolve(config, deps) {
|
|
|
37
38
|
suite_file: item.suite_file,
|
|
38
39
|
runner_manifest: item.runner_manifest,
|
|
39
40
|
}),
|
|
40
|
-
|
|
41
|
+
// The union is never empty here: the caller only asks once the behavioral
|
|
42
|
+
// peer is ready or a feature has checks, so at least one envelope is in it.
|
|
43
|
+
materializeBehavioral: (projectRoot, index, runner) => materializeBehavioralUnion(projectRoot, index, runner),
|
|
41
44
|
runStructural: runStructuralByRunner,
|
|
42
45
|
runBehavioral: runBddSuite,
|
|
43
46
|
validateStack,
|
|
@@ -52,16 +55,24 @@ async function execute(config, d, only) {
|
|
|
52
55
|
const unusable = placeProblem(config.projectRoot) ?? runnerEnvironmentPlaceProblem(config.projectRoot);
|
|
53
56
|
if (unusable)
|
|
54
57
|
throw new Error(`${unusable}\nNothing was run and no results were filed.`);
|
|
55
|
-
const
|
|
56
|
-
const ready = suites.filter((item) => item.status === 'ready');
|
|
58
|
+
const index = await d.getSuiteIndex();
|
|
59
|
+
const ready = index.suites.filter((item) => item.status === 'ready');
|
|
57
60
|
const selected = only === null ? ready : select(ready, only);
|
|
58
|
-
|
|
61
|
+
// The features' checks ride with the behavioral branch: every run of `check`
|
|
62
|
+
// has them, and the first run after publishing has them when the behavioral
|
|
63
|
+
// branch is what was published — structural alone has no union to run on.
|
|
64
|
+
const features = only === null || selected.some((item) => item.suite_kind === 'behavioral') ? index.feature_suites : [];
|
|
65
|
+
if (selected.length === 0 && features.length === 0) {
|
|
59
66
|
d.stdout.write('No Unitbob suites exist yet. Generate them first, then run the Unitbob checks again.\n');
|
|
60
67
|
return;
|
|
61
68
|
}
|
|
69
|
+
const unionFor = unionOnce(config, d, index);
|
|
62
70
|
const runs = [];
|
|
63
71
|
for (const item of selected) {
|
|
64
|
-
runs.push(await buildRunPayload(config, d, item));
|
|
72
|
+
runs.push(await buildRunPayload(config, d, item, unionFor));
|
|
73
|
+
}
|
|
74
|
+
for (const item of features) {
|
|
75
|
+
runs.push(await buildFeatureRunPayload(config, d, item, unionFor));
|
|
65
76
|
}
|
|
66
77
|
const { results, map_url } = await d.postRunsBatch(runs);
|
|
67
78
|
for (const result of results)
|
|
@@ -86,48 +97,88 @@ function select(ready, wanted) {
|
|
|
86
97
|
}
|
|
87
98
|
return wanted.map((digest) => byDigest.get(digest));
|
|
88
99
|
}
|
|
100
|
+
function unionOnce(config, d, index) {
|
|
101
|
+
let union = null;
|
|
102
|
+
return (runner) => {
|
|
103
|
+
if (union)
|
|
104
|
+
return union;
|
|
105
|
+
try {
|
|
106
|
+
union = d.materializeBehavioral(config.projectRoot, index, runner);
|
|
107
|
+
}
|
|
108
|
+
catch (err) {
|
|
109
|
+
// Not a branch's error to file and move past: the union refused to wipe
|
|
110
|
+
// a feature's rewired checks (spec 52-4, AC 1.8). Nothing was run,
|
|
111
|
+
// nothing is posted, and the sentence reaches the terminal through
|
|
112
|
+
// `cli.ts`.
|
|
113
|
+
if (err instanceof FeatureFilesChangedError)
|
|
114
|
+
throw err;
|
|
115
|
+
union = { error: err.message };
|
|
116
|
+
}
|
|
117
|
+
return union;
|
|
118
|
+
};
|
|
119
|
+
}
|
|
89
120
|
// One branch's run payload. A stack mismatch, a materialize failure, or a runner
|
|
90
121
|
// that produced no report all become this branch's structured suite error — the
|
|
91
122
|
// peer branch is unaffected. This connector never installs anything: a missing
|
|
92
123
|
// or broken runner surfaces here as a suite error, not an install.
|
|
93
|
-
async function buildRunPayload(config, d, item) {
|
|
124
|
+
async function buildRunPayload(config, d, item, unionFor) {
|
|
94
125
|
const runner = item.runner_manifest.runner;
|
|
95
|
-
const
|
|
126
|
+
const digest = item.suite_digest;
|
|
96
127
|
// Confirm the local stack before touching the tree, for both contract systems.
|
|
97
128
|
// A mismatch is this branch's suite error — reported and left for the peer
|
|
98
129
|
// branch to run regardless. The behavioral check confirms only the base
|
|
99
130
|
// language; a missing BDD runner still surfaces from the run itself, since
|
|
100
131
|
// check installs nothing.
|
|
101
132
|
const check = d.validateStack(config.projectRoot, runner);
|
|
133
|
+
if (!check.ok)
|
|
134
|
+
return suiteError(digest, check.message ?? `Local project does not match "${runner}".`);
|
|
135
|
+
if (item.suite_kind !== 'behavioral') {
|
|
136
|
+
return filed(runner, digest, async () => {
|
|
137
|
+
d.materializeStructural(config.projectRoot, item);
|
|
138
|
+
return d.runStructural(config.projectRoot, runner, artifactPaths(item.suite_file));
|
|
139
|
+
});
|
|
140
|
+
}
|
|
141
|
+
const union = unionFor(runner);
|
|
142
|
+
if ('error' in union)
|
|
143
|
+
return suiteError(digest, union.error);
|
|
144
|
+
return filed(runner, digest, () => d.runBehavioral(config.projectRoot, runner, union.mainPath, { exclude: union.excludeTags }));
|
|
145
|
+
}
|
|
146
|
+
// One feature's run payload (spec 52-4, AC 1.1): the same checks a branch
|
|
147
|
+
// gets, the same union, only its own tag — and its own suite error when it
|
|
148
|
+
// cannot run, which stops nothing else.
|
|
149
|
+
async function buildFeatureRunPayload(config, d, item, unionFor) {
|
|
150
|
+
const runner = item.runner_manifest.runner;
|
|
151
|
+
const check = d.validateStack(config.projectRoot, runner);
|
|
102
152
|
if (!check.ok)
|
|
103
153
|
return suiteError(item.suite_digest, check.message ?? `Local project does not match "${runner}".`);
|
|
154
|
+
const union = unionFor(runner);
|
|
155
|
+
if ('error' in union)
|
|
156
|
+
return suiteError(item.suite_digest, union.error);
|
|
157
|
+
return filed(runner, item.suite_digest, () => d.runBehavioral(config.projectRoot, runner, union.mainPath, { only: item.feature_tag }));
|
|
158
|
+
}
|
|
159
|
+
// The run itself, filed as this branch's payload: a runner that could not
|
|
160
|
+
// start or a report that cannot be read is its structured suite error.
|
|
161
|
+
async function filed(runner, digest, execute) {
|
|
104
162
|
let result;
|
|
105
163
|
try {
|
|
106
|
-
|
|
107
|
-
const mainPath = d.materializeBehavioral(config.projectRoot, item);
|
|
108
|
-
result = await d.runBehavioral(config.projectRoot, runner, mainPath);
|
|
109
|
-
}
|
|
110
|
-
else {
|
|
111
|
-
d.materializeStructural(config.projectRoot, item);
|
|
112
|
-
result = await d.runStructural(config.projectRoot, runner, artifactPaths(item.suite_file));
|
|
113
|
-
}
|
|
164
|
+
result = await execute();
|
|
114
165
|
}
|
|
115
166
|
catch (err) {
|
|
116
|
-
return suiteError(
|
|
167
|
+
return suiteError(digest, err.message);
|
|
117
168
|
}
|
|
118
169
|
const report = boundReport(runner, result);
|
|
119
170
|
if (report === null) {
|
|
120
171
|
return {
|
|
121
|
-
suite_digest:
|
|
172
|
+
suite_digest: digest,
|
|
122
173
|
suite_error: {
|
|
123
174
|
command: [result.command, ...result.args].join(' '),
|
|
124
175
|
exit_code: result.code,
|
|
125
176
|
result_path: result.resultPath,
|
|
126
|
-
output_tail: outputTail(result),
|
|
177
|
+
output_tail: outputTail(result, OUTPUT_TAIL_CHARS),
|
|
127
178
|
},
|
|
128
179
|
};
|
|
129
180
|
}
|
|
130
|
-
return { suite_digest:
|
|
181
|
+
return { suite_digest: digest, run_result: report };
|
|
131
182
|
}
|
|
132
183
|
// Exported for `run-local`, which runs these same strategies against the files
|
|
133
184
|
// the host just wrote rather than against a published suite. One dispatch table,
|
|
@@ -159,12 +210,3 @@ function suiteError(suiteDigest, message) {
|
|
|
159
210
|
suite_error: { command: '', exit_code: null, result_path: '', output_tail: message },
|
|
160
211
|
};
|
|
161
212
|
}
|
|
162
|
-
function outputTail(result) {
|
|
163
|
-
const bits = [];
|
|
164
|
-
if (result.stderr.trim())
|
|
165
|
-
bits.push(result.stderr.trim());
|
|
166
|
-
if (result.stdout.trim())
|
|
167
|
-
bits.push(result.stdout.trim());
|
|
168
|
-
const joined = bits.join('\n');
|
|
169
|
-
return joined.length > OUTPUT_TAIL_CHARS ? joined.slice(-OUTPUT_TAIL_CHARS) : joined;
|
|
170
|
-
}
|
package/dist/verbs/runLocal.js
CHANGED
|
@@ -5,8 +5,10 @@ import { placeAdvice } from "../runner/placeAdvice.js";
|
|
|
5
5
|
import { runnerEnvironmentPlaceProblem } from "../runner/placeEnvironment.js";
|
|
6
6
|
import { validateStack } from "../runner/precheck.js";
|
|
7
7
|
import { runBddSuite } from "../runner/bdd.js";
|
|
8
|
+
import { parseFeatureId, readTestsRequest } from "../files/features.js";
|
|
8
9
|
import { testPathsOf } from "../runner/vitest.js";
|
|
9
10
|
import { runStructuralByRunner } from "./run.js";
|
|
11
|
+
import { outputTail } from "../runner/outputTail.js";
|
|
10
12
|
const OUTPUT_TAIL_CHARS = 4000;
|
|
11
13
|
export async function runLocal(config, args = [], deps) {
|
|
12
14
|
const d = {
|
|
@@ -23,6 +25,10 @@ export async function runLocal(config, args = [], deps) {
|
|
|
23
25
|
const unusable = placeProblem(config.projectRoot) ?? runnerEnvironmentPlaceProblem(config.projectRoot);
|
|
24
26
|
if (unusable)
|
|
25
27
|
throw new Error(unusable);
|
|
28
|
+
const featureFlag = args.indexOf('--feature');
|
|
29
|
+
if (featureFlag !== -1) {
|
|
30
|
+
return runFeature(config, d, parseFeatureId(args[featureFlag + 1], 'run-local --feature'));
|
|
31
|
+
}
|
|
26
32
|
const request = readSuiteBuildRequest(config.projectRoot);
|
|
27
33
|
const { outputs, unreadable } = readHostSuiteOutputsPerBranch(request.output_path, request);
|
|
28
34
|
const wanted = selectBranches(request, args);
|
|
@@ -39,7 +45,7 @@ export async function runLocal(config, args = [], deps) {
|
|
|
39
45
|
d.stdout.write(`Cannot run this branch — its entry in your answer could not be read: ${broken.message}\n`);
|
|
40
46
|
continue;
|
|
41
47
|
}
|
|
42
|
-
const ran = await runOneBranch(config, d, suiteKind, outputs.find((entry) => entry.suite_kind === suiteKind));
|
|
48
|
+
const ran = await runOneBranch(config, d, suiteKind, outputs.find((entry) => entry.suite_kind === suiteKind), { exclude: request.exclude_feature_tags });
|
|
43
49
|
// A branch with no entry written yet, or one the stack cannot execute,
|
|
44
50
|
// produced nothing to compare: it is the ordinary state halfway through a
|
|
45
51
|
// build, not a repair loop going nowhere.
|
|
@@ -47,9 +53,54 @@ export async function runLocal(config, args = [], deps) {
|
|
|
47
53
|
continue;
|
|
48
54
|
if (compareFailures(config, d, suiteKind, ran, previous[suiteKind]))
|
|
49
55
|
stuck = true;
|
|
56
|
+
// The features' checks, each by its tag, on the same files. No stall
|
|
57
|
+
// comparison for them: their loop is the feature's own, in `--feature`.
|
|
58
|
+
if (suiteKind === 'behavioral') {
|
|
59
|
+
for (const tag of request.exclude_feature_tags)
|
|
60
|
+
await runTag(config, d, ran.runner, ran.mainPath, tag);
|
|
61
|
+
}
|
|
50
62
|
}
|
|
51
63
|
return stuck ? 1 : 0;
|
|
52
64
|
}
|
|
65
|
+
async function runTag(config, d, runner, mainPath, tag) {
|
|
66
|
+
d.stdout.write(`\n── ${tag} ──\n`);
|
|
67
|
+
let result;
|
|
68
|
+
try {
|
|
69
|
+
result = await d.runBehavioral(config.projectRoot, runner, mainPath, { only: tag });
|
|
70
|
+
}
|
|
71
|
+
catch (err) {
|
|
72
|
+
const advice = placeAdvice(config.projectRoot);
|
|
73
|
+
d.stdout.write(`The runner could not start: ${err.message}\n${advice ? `\n${advice}\n` : ''}`);
|
|
74
|
+
return;
|
|
75
|
+
}
|
|
76
|
+
d.stdout.write(report(result));
|
|
77
|
+
d.stdout.write(failureLines(runner, result));
|
|
78
|
+
}
|
|
79
|
+
// One feature's checks, by their tag (spec 52-3, AC 3.3). No stall comparison:
|
|
80
|
+
// the loop here ends on "every scenario red", which the person reads off the
|
|
81
|
+
// failures printed, and the server's proof of red is the connector's own run
|
|
82
|
+
// in `put-tests`, not this one.
|
|
83
|
+
async function runFeature(config, d, featureId) {
|
|
84
|
+
const request = readTestsRequest(config.projectRoot, featureId);
|
|
85
|
+
d.stdout.write(`\n── feature ${featureId} ──\n`);
|
|
86
|
+
const check = d.validateStack(config.projectRoot, request.runner);
|
|
87
|
+
if (!check.ok) {
|
|
88
|
+
d.stdout.write(`Cannot run the checks: ${check.message ?? `this project does not match "${request.runner}".`}\n`);
|
|
89
|
+
return 1;
|
|
90
|
+
}
|
|
91
|
+
let result;
|
|
92
|
+
try {
|
|
93
|
+
result = await d.runBehavioral(config.projectRoot, request.runner, request.feature_path, { only: request.feature_tag });
|
|
94
|
+
}
|
|
95
|
+
catch (err) {
|
|
96
|
+
const advice = placeAdvice(config.projectRoot);
|
|
97
|
+
d.stdout.write(`The runner could not start: ${err.message}\n${advice ? `\n${advice}\n` : ''}`);
|
|
98
|
+
return 1;
|
|
99
|
+
}
|
|
100
|
+
d.stdout.write(report(result));
|
|
101
|
+
d.stdout.write(failureLines(request.runner, result));
|
|
102
|
+
return 0;
|
|
103
|
+
}
|
|
53
104
|
// Spec 34-6, criterion 3. The whole stop condition, and it stops the branch
|
|
54
105
|
// rather than the worker: the set of failures belongs to the branch, and a
|
|
55
106
|
// repair worker looking only at its own slice cannot see that the branch as a
|
|
@@ -97,7 +148,7 @@ function selectBranches(request, args) {
|
|
|
97
148
|
// Non-null when the runner actually executed the branch. Everything else — no
|
|
98
149
|
// entry, a declared `build_error`, a stack that cannot run it — is a branch that
|
|
99
150
|
// produced no result to compare against.
|
|
100
|
-
async function runOneBranch(config, d, suiteKind, output) {
|
|
151
|
+
async function runOneBranch(config, d, suiteKind, output, filter) {
|
|
101
152
|
// Nothing written for this branch yet. That is the ordinary state halfway
|
|
102
153
|
// through a build, not an error — say what is missing and move to the peer.
|
|
103
154
|
if (!output) {
|
|
@@ -128,7 +179,7 @@ async function runOneBranch(config, d, suiteKind, output) {
|
|
|
128
179
|
try {
|
|
129
180
|
result =
|
|
130
181
|
suiteKind === 'behavioral'
|
|
131
|
-
? await d.runBehavioral(config.projectRoot, runner, suitePaths[0])
|
|
182
|
+
? await d.runBehavioral(config.projectRoot, runner, suitePaths[0], filter)
|
|
132
183
|
: await d.runStructural(config.projectRoot, runner, suitePaths);
|
|
133
184
|
}
|
|
134
185
|
catch (err) {
|
|
@@ -142,7 +193,7 @@ async function runOneBranch(config, d, suiteKind, output) {
|
|
|
142
193
|
}
|
|
143
194
|
d.stdout.write(report(result));
|
|
144
195
|
d.stdout.write(failureLines(runner, result));
|
|
145
|
-
return { runner, result };
|
|
196
|
+
return { runner, mainPath: suitePaths[0], result };
|
|
146
197
|
}
|
|
147
198
|
// How many lines of one failure's message are worth printing here. Enough for an
|
|
148
199
|
// assertion diff and the frame under it; not the whole backtrace, which is in
|
|
@@ -224,20 +275,11 @@ function report(result) {
|
|
|
224
275
|
lines.push(`no report at ${result.resultPath} — the run produced none, which usually means it died before` +
|
|
225
276
|
' the first test rather than that the tests failed.');
|
|
226
277
|
}
|
|
227
|
-
const tail = outputTail(result);
|
|
278
|
+
const tail = outputTail(result, OUTPUT_TAIL_CHARS);
|
|
228
279
|
if (tail)
|
|
229
280
|
lines.push('', tail);
|
|
230
281
|
return `${lines.join('\n')}\n`;
|
|
231
282
|
}
|
|
232
|
-
function outputTail(result) {
|
|
233
|
-
const bits = [];
|
|
234
|
-
if (result.stderr.trim())
|
|
235
|
-
bits.push(result.stderr.trim());
|
|
236
|
-
if (result.stdout.trim())
|
|
237
|
-
bits.push(result.stdout.trim());
|
|
238
|
-
const joined = bits.join('\n');
|
|
239
|
-
return joined.length > OUTPUT_TAIL_CHARS ? joined.slice(-OUTPUT_TAIL_CHARS) : joined;
|
|
240
|
-
}
|
|
241
283
|
// The suite blob's own project-relative paths, exactly as the runners expect
|
|
242
284
|
// them: the main file first, then every other file of the branch. The main file
|
|
243
285
|
// stopped being the whole suite in spec one-place-per-rule, §6 — a branch is one file per
|
|
@@ -66,6 +66,7 @@ export async function suitePrepare(config, args = [], deps) {
|
|
|
66
66
|
const actual = {
|
|
67
67
|
getRecipe: (name) => wire.getRecipe(name),
|
|
68
68
|
getSuitePacketsBatch: () => wire.getSuitePacketsBatch(),
|
|
69
|
+
getSuiteIndex: () => wire.getSuiteIndex(),
|
|
69
70
|
precheck: anyStackPrecheck,
|
|
70
71
|
confirmRunner: (projectRoot, runner) => runnerReadyPrecheck(projectRoot, runner),
|
|
71
72
|
bootCheck: (projectRoot, runner, sourceFiles) => bootCheck(projectRoot, runner, sourceFiles),
|
|
@@ -314,7 +315,8 @@ export async function suitePrepare(config, args = [], deps) {
|
|
|
314
315
|
const why = [...bootNotices, ...blockedNotices, ...fixableNotices].join('\n');
|
|
315
316
|
throw new Error(`No suite branch can be built this run:\n${why}\nNothing was written and nothing was uploaded.`);
|
|
316
317
|
}
|
|
317
|
-
const
|
|
318
|
+
const excludeFeatureTags = (await actual.getSuiteIndex()).feature_suites.map((item) => item.feature_tag);
|
|
319
|
+
const request = writeSuiteBuildRequest(config.projectRoot, branches, defectContext, excludeFeatureTags);
|
|
318
320
|
// Spec 37-1. The assignment names entrypoints; the packets are the files
|
|
319
321
|
// behind them, resolved from this machine's own graph and copied where a
|
|
320
322
|
// worker can open them. Built here because the entrypoints are known from the
|
|
@@ -2,13 +2,21 @@ import { execFileSync } from 'node:child_process';
|
|
|
2
2
|
import { mkdtempSync, rmSync } from 'node:fs';
|
|
3
3
|
import { tmpdir } from 'node:os';
|
|
4
4
|
import { join } from 'node:path';
|
|
5
|
-
import { copyBehavioralRunnerEnvironment, filesLostOnMaterialize,
|
|
5
|
+
import { changedFeatureFiles, copyBehavioralRunnerEnvironment, FeatureFilesChangedError, filesLostOnMaterialize, materializeBehavioralUnion, } from "../files/behavioral.js";
|
|
6
6
|
import { runBddSuite } from "../runner/bdd.js";
|
|
7
7
|
import { boundReport } from "../runner/boundReport.js";
|
|
8
8
|
import { placeOf } from "../runner/place.js";
|
|
9
|
+
import { gitRevision } from "../runner/gitRevision.js";
|
|
10
|
+
import { Wire } from "../wire.js";
|
|
9
11
|
import { branchRunner, readHostSuiteOutputs, readSuiteBuildRequest, reviewRequestPath, writeBehavioralReviewRequest, } from "../files/suiteBuild.js";
|
|
12
|
+
// The candidate is run on disk as the same union `check` writes (spec 52-4,
|
|
13
|
+
// AC 1.8): the candidate plus the checks of every red feature the server
|
|
14
|
+
// holds, with their tags left out of the run. Written as the candidate alone,
|
|
15
|
+
// the review wiped those checks from the disk — and with them any steps the
|
|
16
|
+
// host had rewired against the real code and not yet saved.
|
|
10
17
|
export async function suiteReviewPrepare(config, _args = [], deps) {
|
|
11
18
|
const actual = {
|
|
19
|
+
getSuiteIndex: () => new Wire(config).getSuiteIndex(),
|
|
12
20
|
runCandidate: runCandidate,
|
|
13
21
|
stdout: process.stdout,
|
|
14
22
|
...deps,
|
|
@@ -21,40 +29,63 @@ export async function suiteReviewPrepare(config, _args = [], deps) {
|
|
|
21
29
|
if (behavioral.build_error) {
|
|
22
30
|
throw new Error(`The behavioral candidate could not be reviewed: ${behavioral.build_error.message}`);
|
|
23
31
|
}
|
|
32
|
+
// Before anything else is said or run: a feature's checks rewired on disk
|
|
33
|
+
// and not saved stop the review here, disk untouched, with the command that
|
|
34
|
+
// saves them — the same stop `check` makes.
|
|
35
|
+
const features = (await actual.getSuiteIndex()).feature_suites;
|
|
36
|
+
const changed = changedFeatureFiles(config.projectRoot, features);
|
|
37
|
+
if (changed.length > 0)
|
|
38
|
+
throw new FeatureFilesChangedError(changed[0].feature_id, changed[0].title);
|
|
24
39
|
// Say this before the run, not after: the run materializes the answer, and
|
|
25
40
|
// that is where a forgotten file turns into undefined steps — by then it is
|
|
26
41
|
// already gone.
|
|
27
|
-
const lost = filesLostOnMaterialize(config.projectRoot, behavioral.suite_file, branchRunner(behavioral));
|
|
42
|
+
const lost = filesLostOnMaterialize(config.projectRoot, behavioral.suite_file, branchRunner(behavioral), features.map((item) => item.suite_file));
|
|
28
43
|
if (lost.length > 0) {
|
|
29
44
|
actual.stdout.write(`Warning: these files sit with the suite but are not in its answer, and running it will delete them: ${lost.join(', ')}.\n`);
|
|
30
45
|
}
|
|
31
|
-
const candidateRun = await actual.runCandidate(config.projectRoot, behavioral);
|
|
46
|
+
const candidateRun = await actual.runCandidate(config.projectRoot, behavioral, features);
|
|
32
47
|
const fixedRevision = buildRequest.known_defect_context.status === 'supplied'
|
|
33
48
|
? buildRequest.known_defect_context.fixed_revision
|
|
34
49
|
: undefined;
|
|
35
50
|
const fixedCandidateRun = fixedRevision
|
|
36
|
-
? await actual.runCandidate(config.projectRoot, behavioral, fixedRevision)
|
|
51
|
+
? await actual.runCandidate(config.projectRoot, behavioral, features, fixedRevision)
|
|
37
52
|
: undefined;
|
|
38
53
|
const request = writeBehavioralReviewRequest(config.projectRoot, behavioral, candidateRun, buildRequest.known_defect_context, fixedCandidateRun);
|
|
39
54
|
actual.stdout.write(`Behavioral review request written to ${reviewRequestPath(config.projectRoot)}\n`);
|
|
40
55
|
actual.stdout.write(`Next: have an independent reviewer inspect this exact candidate and write bdd_quality_review and known_defect_probe to ${request.output_path}, then run \`unitbob put-suite-build\`.\n`);
|
|
41
56
|
}
|
|
42
|
-
async function runCandidate(projectRoot, output, revision) {
|
|
57
|
+
async function runCandidate(projectRoot, output, features, revision) {
|
|
43
58
|
if (revision)
|
|
44
|
-
return runCandidateAtRevision(projectRoot, output, revision);
|
|
45
|
-
return runCandidateInProject(projectRoot, output, gitRevision(projectRoot));
|
|
59
|
+
return runCandidateAtRevision(projectRoot, output, features, revision);
|
|
60
|
+
return runCandidateInProject(projectRoot, output, features, gitRevision(projectRoot));
|
|
46
61
|
}
|
|
47
|
-
async function runCandidateInProject(projectRoot, output, revision) {
|
|
62
|
+
async function runCandidateInProject(projectRoot, output, features, revision) {
|
|
48
63
|
const runner = branchRunner(output);
|
|
49
|
-
const
|
|
50
|
-
const
|
|
51
|
-
const result = await runBddSuite(projectRoot, runner, mainPath);
|
|
64
|
+
const { mainPath, excludeTags } = candidateUnion(projectRoot, output, features);
|
|
65
|
+
const result = await runBddSuite(projectRoot, runner, mainPath, { exclude: excludeTags });
|
|
52
66
|
const report = boundReport(runner, result);
|
|
53
67
|
if (report === null)
|
|
54
68
|
throw new Error('The behavioral candidate produced no machine-readable runner report.');
|
|
55
69
|
return { revision, run_result: report };
|
|
56
70
|
}
|
|
57
|
-
|
|
71
|
+
// The candidate on disk as `check` would write it: the union of the candidate
|
|
72
|
+
// and every feature's checks, one clearing, the feature tags to leave out.
|
|
73
|
+
// Through `materializeBehavioralUnion` rather than beside it, so the stop on
|
|
74
|
+
// a changed feature file is the same one, made before the disk is touched.
|
|
75
|
+
export function candidateUnion(projectRoot, output, features) {
|
|
76
|
+
const index = {
|
|
77
|
+
suites: [{
|
|
78
|
+
suite_kind: 'behavioral',
|
|
79
|
+
status: 'ready',
|
|
80
|
+
suite_file: output.suite_file,
|
|
81
|
+
runner_manifest: output.runner_manifest,
|
|
82
|
+
}],
|
|
83
|
+
feature_suites: [...features],
|
|
84
|
+
};
|
|
85
|
+
// Never null: the candidate itself is in the union.
|
|
86
|
+
return materializeBehavioralUnion(projectRoot, index, branchRunner(output));
|
|
87
|
+
}
|
|
88
|
+
async function runCandidateAtRevision(projectRoot, output, features, revision) {
|
|
58
89
|
// Spec 36, Non-Goals. The worktree below is created under the system's
|
|
59
90
|
// temporary directory — outside anything a container has mounted, so in there
|
|
60
91
|
// it does not exist at all. Said plainly rather than run into. The obvious
|
|
@@ -78,9 +109,9 @@ async function runCandidateAtRevision(projectRoot, output, revision) {
|
|
|
78
109
|
execFileSync('git', ['worktree', 'add', '--detach', worktree, resolved], { cwd: projectRoot, stdio: 'pipe' });
|
|
79
110
|
added = true;
|
|
80
111
|
const runner = branchRunner(output);
|
|
81
|
-
|
|
112
|
+
candidateUnion(worktree, output, features);
|
|
82
113
|
copyBehavioralRunnerEnvironment(projectRoot, worktree, runner);
|
|
83
|
-
return await runCandidateInProject(worktree, output, revision);
|
|
114
|
+
return await runCandidateInProject(worktree, output, features, revision);
|
|
84
115
|
}
|
|
85
116
|
finally {
|
|
86
117
|
if (added) {
|
|
@@ -105,18 +136,3 @@ async function runCandidateAtRevision(projectRoot, output, revision) {
|
|
|
105
136
|
}
|
|
106
137
|
}
|
|
107
138
|
}
|
|
108
|
-
function gitRevision(projectRoot) {
|
|
109
|
-
try {
|
|
110
|
-
const options = {
|
|
111
|
-
cwd: projectRoot,
|
|
112
|
-
encoding: 'utf8',
|
|
113
|
-
stdio: ['ignore', 'pipe', 'pipe'],
|
|
114
|
-
};
|
|
115
|
-
const head = execFileSync('git', ['rev-parse', 'HEAD'], options).trim();
|
|
116
|
-
const dirty = execFileSync('git', ['status', '--porcelain'], options).trim();
|
|
117
|
-
return dirty ? `${head}-dirty` : head;
|
|
118
|
-
}
|
|
119
|
-
catch {
|
|
120
|
-
return 'working-tree';
|
|
121
|
-
}
|
|
122
|
-
}
|