unitbob 0.4.4 → 0.5.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli.js +4 -2
- package/dist/files/guardrails.js +19 -9
- package/dist/files/suiteBuild.js +20 -4
- package/dist/files/suiteBuildUpload.js +71 -0
- package/dist/files/workerPlan.js +33 -19
- package/dist/proc.js +15 -0
- package/dist/runner/bootcheck.js +47 -9
- package/dist/runner/failureDigest.js +238 -0
- package/dist/runner/precheck.js +104 -36
- package/dist/runner/provision.js +326 -26
- package/dist/runner/pytest.js +25 -20
- package/dist/runner/rspec.js +19 -11
- package/dist/runner/toolchain.js +122 -0
- package/dist/runner/vitest.js +64 -30
- package/dist/surfaces/routeInventory.js +15 -3
- package/dist/verbs/codexInstall.js +1 -1
- package/dist/verbs/putSuiteBuild.js +32 -75
- package/dist/verbs/run.js +15 -6
- package/dist/verbs/runLocal.js +69 -23
- package/dist/verbs/suitePrepare.js +49 -15
- package/dist/verbs/suiteReviewPrepare.js +0 -22
- package/dist/verbs/validateBuild.js +140 -450
- package/dist/wire.js +21 -3
- package/package.json +1 -1
- package/plugin/codex/agents/suite-repair-worker.toml +18 -6
- package/plugin/codex/agents/suite-reviewer.toml +157 -0
- package/plugin/codex/agents/suite-worker.toml +37 -20
- package/dist/files/budget.js +0 -74
package/dist/cli.js
CHANGED
|
@@ -133,8 +133,10 @@ export async function main(argv, deps = { ensureLinked }) {
|
|
|
133
133
|
await contractPrompt(await linked(), args);
|
|
134
134
|
return 0;
|
|
135
135
|
case 'run-local':
|
|
136
|
-
|
|
137
|
-
|
|
136
|
+
// The one verb whose non-zero exit is not an error: a branch that failed
|
|
137
|
+
// the same set of cases twice in a row (spec 34-6, criterion 3). Red
|
|
138
|
+
// tests still exit zero — a live defect is the suite working.
|
|
139
|
+
return await runLocal(await linked(), args);
|
|
138
140
|
case 'run':
|
|
139
141
|
case 'check':
|
|
140
142
|
await run(await linked(), args);
|
package/dist/files/guardrails.js
CHANGED
|
@@ -62,21 +62,31 @@ else
|
|
|
62
62
|
end
|
|
63
63
|
abort 'unitbob_helper: refusing to run against a non-test environment' unless Rails.env.test?
|
|
64
64
|
`;
|
|
65
|
-
// Write the suite blob
|
|
66
|
-
//
|
|
67
|
-
//
|
|
68
|
-
//
|
|
65
|
+
// Write every file of the suite blob at its own (validated) relative path. The
|
|
66
|
+
// Ruby boot kit is materialized only for the rspec runner — Vitest and pytest
|
|
67
|
+
// runs need no connector-written support files here (the runtime pytest.ini
|
|
68
|
+
// lives outside this directory and is written by the pytest runner).
|
|
69
|
+
//
|
|
70
|
+
// Every file, not just the main one (spec 42, §6.4). The directory is wiped
|
|
71
|
+
// first and only the main file was written back, so a published suite of four
|
|
72
|
+
// files came back as one and the run that followed it silently protected a
|
|
73
|
+
// quarter of what the map claimed.
|
|
69
74
|
export function materializeGuardrails(projectRoot, suite) {
|
|
70
|
-
|
|
75
|
+
const files = [suite.suite_file, ...(suite.suite_file.support_files ?? [])];
|
|
76
|
+
for (const file of files)
|
|
77
|
+
assertGuardrailPath(file.path);
|
|
71
78
|
const dir = join(projectRoot, GUARDRAILS_DIR);
|
|
72
79
|
rmSync(dir, { recursive: true, force: true });
|
|
73
80
|
mkdirSync(dir, { recursive: true });
|
|
74
|
-
const
|
|
75
|
-
|
|
76
|
-
|
|
81
|
+
const written = files.map((file) => {
|
|
82
|
+
const path = join(projectRoot, file.path);
|
|
83
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
84
|
+
writeFileSync(path, file.content);
|
|
85
|
+
return path;
|
|
86
|
+
});
|
|
77
87
|
if (suite.runner_manifest.runner === 'rspec')
|
|
78
88
|
materializeHelper(projectRoot);
|
|
79
|
-
return { suitePath };
|
|
89
|
+
return { suitePath: written[0], supportPaths: written.slice(1) };
|
|
80
90
|
}
|
|
81
91
|
// Both Ruby flows boot the same way: the check flow writes the boot kit next to
|
|
82
92
|
// the suite here, the suite-build flow writes it right after the precheck.
|
package/dist/files/suiteBuild.js
CHANGED
|
@@ -2,7 +2,6 @@ import { existsSync, mkdirSync, readFileSync, realpathSync, writeFileSync } from
|
|
|
2
2
|
import { createHash } from 'node:crypto';
|
|
3
3
|
import { dirname, join, sep } from 'node:path';
|
|
4
4
|
import { assertUnitbobPath } from "./artifactPath.js";
|
|
5
|
-
import { readBudget, RUN_BUDGET } from "./budget.js";
|
|
6
5
|
import { readWorkerPlan, validateWorkerPlanFiles, workerPlanDigest, workerPlanPath } from "./workerPlan.js";
|
|
7
6
|
export function requestPath(projectRoot) {
|
|
8
7
|
return join(projectRoot, '.unitbob', 'suite-build', 'request.json');
|
|
@@ -83,12 +82,27 @@ export function branchRunner(output) {
|
|
|
83
82
|
}
|
|
84
83
|
return runner;
|
|
85
84
|
}
|
|
85
|
+
// What the reviewer actually read: the suite files, and the manifest that runs
|
|
86
|
+
// them. Nothing else (spec 42, §4).
|
|
87
|
+
//
|
|
88
|
+
// `test_metadata` used to be in here, and the server's copy of this formula
|
|
89
|
+
// stripped the review's own keys back out to match — two lists that had to stay
|
|
90
|
+
// identical for ever or every upload would break. The real cost was elsewhere,
|
|
91
|
+
// though: editing metadata declared the review stale. On noahsat-web,
|
|
92
|
+
// 2026-08-12 the reviewer was right that the steps drive only `PATCH`, the fix
|
|
93
|
+
// moved three `PUT` aliases into a deferred list, not one byte of the suite
|
|
94
|
+
// moved — and the run still paid for re-binding the candidate and a second
|
|
95
|
+
// reviewer pass, the most expensive step of the whole recipe, to satisfy the
|
|
96
|
+
// reviewer's own finding.
|
|
97
|
+
//
|
|
98
|
+
// `stableJson` sorts object keys and does nothing else. The server's
|
|
99
|
+
// `canonical_json` does the same, which is the only reason the two sides agree;
|
|
100
|
+
// a normalization added on one side alone would break every upload.
|
|
86
101
|
export function suiteCandidateDigest(output) {
|
|
87
102
|
return createHash('sha256')
|
|
88
103
|
.update(stableJson({
|
|
89
104
|
suite_file: output.suite_file,
|
|
90
105
|
runner_manifest: output.runner_manifest,
|
|
91
|
-
test_metadata: output.test_metadata,
|
|
92
106
|
}))
|
|
93
107
|
.digest('hex');
|
|
94
108
|
}
|
|
@@ -232,7 +246,6 @@ export function writeSuiteBuildRequest(projectRoot, branches, knownDefectContext
|
|
|
232
246
|
output_path: outputPath(projectRoot),
|
|
233
247
|
branches,
|
|
234
248
|
known_defect_context: knownDefectContext,
|
|
235
|
-
budget: RUN_BUDGET,
|
|
236
249
|
};
|
|
237
250
|
const path = requestPath(projectRoot);
|
|
238
251
|
mkdirSync(dirname(path), { recursive: true });
|
|
@@ -251,10 +264,13 @@ export function readSuiteBuildRequest(projectRoot) {
|
|
|
251
264
|
!Array.isArray(request.branches)) {
|
|
252
265
|
throw new Error(`${path} is malformed: expected project_root, output_path, and a branches array.`);
|
|
253
266
|
}
|
|
267
|
+
// Spec 34-6, criterion 2.3. A request written by an older connector still
|
|
268
|
+
// carries a `budget` block; it is spread through untouched and read by nobody,
|
|
269
|
+
// which is the whole of the compatibility story — there is no ceiling left for
|
|
270
|
+
// it to name.
|
|
254
271
|
return {
|
|
255
272
|
...request,
|
|
256
273
|
known_defect_context: readKnownDefectContext(request.known_defect_context, path),
|
|
257
|
-
budget: readBudget(request.budget),
|
|
258
274
|
};
|
|
259
275
|
}
|
|
260
276
|
function readKnownDefectContext(value, path) {
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
import { readBehavioralReview } from "./suiteBuild.js";
|
|
2
|
+
// What travels to the server, and what "published" means when it answers. One
|
|
3
|
+
// module, because two commands ask those questions: `put-suite-build` sends the
|
|
4
|
+
// batch, and `validate-build` sends the same batch as a dry run so the server's
|
|
5
|
+
// verdict is about the exact bytes the publish will carry (spec 42, §3).
|
|
6
|
+
//
|
|
7
|
+
// A second assembly would be a second answer to "what are we uploading", and the
|
|
8
|
+
// dry run would then be checking something the publish does not send — which is
|
|
9
|
+
// worth less than not checking at all, because it reads as a verdict.
|
|
10
|
+
// The three outcomes that leave a branch published and current: a new version, an
|
|
11
|
+
// identical version already stored, or a reactivated one. Each returns the
|
|
12
|
+
// identity to run. Everything else — a rejected branch, a branch the host could
|
|
13
|
+
// not build, or a status this connector has never seen — fails closed and is
|
|
14
|
+
// never run, so a newer server can never trick an older connector into running
|
|
15
|
+
// something it does not understand.
|
|
16
|
+
export const PUBLISHED = new Set(['created', 'unchanged', 'restored']);
|
|
17
|
+
// The answer a dry run gives to a branch it would accept. A server that does not
|
|
18
|
+
// know `dry_run` answers one of `PUBLISHED` instead — which means it published —
|
|
19
|
+
// and `validate-build` says so rather than reporting a check that passed.
|
|
20
|
+
export const WOULD_PUBLISH = 'would_publish';
|
|
21
|
+
// One branch, as the upload sends it. `source_digest` comes from the request,
|
|
22
|
+
// never from the host's answer, so the host cannot claim a different map than
|
|
23
|
+
// the branch was given.
|
|
24
|
+
export function uploadItem(request, output, testMetadata) {
|
|
25
|
+
const sourceDigest = request.branches.find((branch) => branch.suite_kind === output.suite_kind)?.source_digest ?? '';
|
|
26
|
+
if (output.build_error) {
|
|
27
|
+
return { suite_kind: output.suite_kind, source_digest: sourceDigest, build_error: output.build_error };
|
|
28
|
+
}
|
|
29
|
+
return {
|
|
30
|
+
suite_kind: output.suite_kind,
|
|
31
|
+
source_digest: sourceDigest,
|
|
32
|
+
artifacts: {
|
|
33
|
+
suite_file: output.suite_file,
|
|
34
|
+
runner_manifest: output.runner_manifest,
|
|
35
|
+
test_metadata: testMetadata,
|
|
36
|
+
},
|
|
37
|
+
};
|
|
38
|
+
}
|
|
39
|
+
// The behavioral branch's uploaded metadata, with the independent review and the
|
|
40
|
+
// connector's own run evidence folded in.
|
|
41
|
+
//
|
|
42
|
+
// Throws for anything that leaves this branch unpublishable — a missing review,
|
|
43
|
+
// one bound to a different candidate, a defect the review called not_supplied.
|
|
44
|
+
// The caller turns that into one unpublished branch rather than a failed
|
|
45
|
+
// command: a blocked review is a fact about the behavioral suite, and the
|
|
46
|
+
// structural peer next to it is finished and correct. Sinking the whole upload
|
|
47
|
+
// with it forced the one workaround this contract exists to prevent — hand-editing
|
|
48
|
+
// the answer down to a single branch, which loses the peer candidate for real.
|
|
49
|
+
export function withReview(config, request, output) {
|
|
50
|
+
const review = readBehavioralReview(config.projectRoot, output);
|
|
51
|
+
const probe = review.known_defect_probe;
|
|
52
|
+
const qualityReview = review.bdd_quality_review;
|
|
53
|
+
if (!qualityReview || typeof qualityReview !== 'object') {
|
|
54
|
+
throw new Error('The separate behavioral review must contain a bdd_quality_review object.');
|
|
55
|
+
}
|
|
56
|
+
if (request.known_defect_context.status === 'supplied' && probe?.status === 'not_supplied') {
|
|
57
|
+
throw new Error('A known defect was supplied to suite-prepare, but the behavioral review marked it not_supplied.');
|
|
58
|
+
}
|
|
59
|
+
return {
|
|
60
|
+
...output.test_metadata,
|
|
61
|
+
bdd_quality_review: {
|
|
62
|
+
...qualityReview,
|
|
63
|
+
candidate_digest: review.candidate_digest,
|
|
64
|
+
},
|
|
65
|
+
...(review.selection_review ? { selection_review: review.selection_review } : {}),
|
|
66
|
+
known_defect_probe: review.known_defect_probe,
|
|
67
|
+
known_defect_context: request.known_defect_context,
|
|
68
|
+
candidate_run: review.candidate_run,
|
|
69
|
+
...(review.fixed_candidate_run ? { fixed_candidate_run: review.fixed_candidate_run } : {}),
|
|
70
|
+
};
|
|
71
|
+
}
|
package/dist/files/workerPlan.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { createHash } from 'node:crypto';
|
|
2
2
|
import { existsSync, readFileSync } from 'node:fs';
|
|
3
3
|
import { join } from 'node:path';
|
|
4
|
+
import { detectStructuralRunner } from "../runner/precheck.js";
|
|
4
5
|
import { assertUnitbobPath } from "./artifactPath.js";
|
|
5
6
|
export function workerPlanPath(projectRoot) {
|
|
6
7
|
return join(projectRoot, '.unitbob', 'suite-build', 'worker-plan.json');
|
|
@@ -33,8 +34,13 @@ export function readWorkerPlan(projectRoot) {
|
|
|
33
34
|
}
|
|
34
35
|
return parsed;
|
|
35
36
|
}
|
|
37
|
+
const RUBY_HARNESS = {
|
|
38
|
+
behavioral: '.unitbob/behavioral/step_definitions/00_unitbob_world.rb',
|
|
39
|
+
structural: '.unitbob/structural/unitbob_helper.rb',
|
|
40
|
+
};
|
|
36
41
|
export function validateWorkerPlanFiles(projectRoot) {
|
|
37
42
|
const errors = [];
|
|
43
|
+
const rubyProject = detectStructuralRunner(projectRoot) === 'rspec';
|
|
38
44
|
const plan = readWorkerPlan(projectRoot);
|
|
39
45
|
let request;
|
|
40
46
|
try {
|
|
@@ -54,8 +60,6 @@ export function validateWorkerPlanFiles(projectRoot) {
|
|
|
54
60
|
branch.suite_kind,
|
|
55
61
|
assignmentIds(branch.assignment),
|
|
56
62
|
]));
|
|
57
|
-
const budget = request.budget;
|
|
58
|
-
const workerCeiling = typeof budget?.workers === 'number' ? budget.workers : Number.POSITIVE_INFINITY;
|
|
59
63
|
const seenWorkers = new Set();
|
|
60
64
|
const seenPaths = new Map();
|
|
61
65
|
for (const [index, item] of plan.workers.entries()) {
|
|
@@ -88,18 +92,26 @@ export function validateWorkerPlanFiles(projectRoot) {
|
|
|
88
92
|
errors.push(`${label}: done_when must be non-empty`);
|
|
89
93
|
if (!isNonEmptyString(item?.harness_path))
|
|
90
94
|
errors.push(`${label}: harness_path must be non-empty`);
|
|
91
|
-
|
|
92
|
-
|
|
93
|
-
|
|
95
|
+
// Both connector-owned harness files are Ruby, and only a Ruby project has
|
|
96
|
+
// them: `unitbob_helper.rb` boots Rails for RSpec, and the behavioral World
|
|
97
|
+
// is materialized for cucumber alone. Demanding them everywhere refused
|
|
98
|
+
// every Python and JS plan over a file that does not exist and would mean
|
|
99
|
+
// nothing if it did — the same mistake as the stack gate that reported a
|
|
100
|
+
// Python project as no stack at all. Found 2026-08-12.
|
|
101
|
+
//
|
|
102
|
+
// What is required of the other stacks is what the rule was ever about: the
|
|
103
|
+
// harness is connector territory, under `.unitbob/`, not a file in the
|
|
104
|
+
// project.
|
|
105
|
+
const expectedHarness = rubyProject ? RUBY_HARNESS[item?.branch] ?? null : null;
|
|
94
106
|
if (expectedHarness && item.harness_path !== expectedHarness) {
|
|
95
107
|
errors.push(`${label}: harness_path must name the connector-owned ${expectedHarness}`);
|
|
96
108
|
}
|
|
109
|
+
if (!expectedHarness && isNonEmptyString(item?.harness_path) && !item.harness_path.startsWith('.unitbob/')) {
|
|
110
|
+
errors.push(`${label}: harness_path must be a connector-owned path under .unitbob/ (got "${item.harness_path}")`);
|
|
111
|
+
}
|
|
97
112
|
if (!item?.limits || item.limits.planned_cases !== item.planned_cases?.length) {
|
|
98
113
|
errors.push(`${label}: limits.planned_cases must equal planned_cases.length`);
|
|
99
114
|
}
|
|
100
|
-
if (!Number.isInteger(item?.limits?.fact_finder_lookups) || item.limits.fact_finder_lookups < 0 || item.limits.fact_finder_lookups > 8) {
|
|
101
|
-
errors.push(`${label}: limits.fact_finder_lookups must be an integer from 0 to 8`);
|
|
102
|
-
}
|
|
103
115
|
for (const ownedPath of Array.isArray(item?.owned_paths) ? item.owned_paths : []) {
|
|
104
116
|
if (!isNonEmptyString(ownedPath)) {
|
|
105
117
|
errors.push(`${label}: owned path must be a non-empty string`);
|
|
@@ -118,27 +130,29 @@ export function validateWorkerPlanFiles(projectRoot) {
|
|
|
118
130
|
seenPaths.set(ownedPath, label);
|
|
119
131
|
}
|
|
120
132
|
}
|
|
133
|
+
// Spec 34-6, criterion 1.6. What is left here catches a corrupted plan, never
|
|
134
|
+
// a small one. The gate used to also demand that every assigned capability
|
|
135
|
+
// appear — which made the plan's size a copy of the assignment's size, and the
|
|
136
|
+
// assignment is the whole product map. That is the load one worker could not
|
|
137
|
+
// finish on a2time, 2026-08-10. Scope is chosen by the coordinator with the
|
|
138
|
+
// user (workflow step 3), and the plan is the only record of it, so a plan
|
|
139
|
+
// that covers part of the assignment is now an ordinary plan.
|
|
140
|
+
//
|
|
141
|
+
// The neighbours stay for the opposite reason: naming a capability the request
|
|
142
|
+
// never assigned, or naming one twice, are ways a plan is wrong rather than
|
|
143
|
+
// ways it is narrow. So is an empty plan for a branch that was given work.
|
|
121
144
|
for (const [branch, expected] of expectedByBranch) {
|
|
122
145
|
const items = plan.workers.filter((item) => item && typeof item === 'object' && !Array.isArray(item) && item.branch === branch);
|
|
123
146
|
if (expected.length > 0 && items.length === 0)
|
|
124
147
|
errors.push(`${branch}: no worker slice was planned`);
|
|
125
|
-
if (items.length > Math.min(workerCeiling, expected.length)) {
|
|
126
|
-
errors.push(`${branch}: worker count ${items.length} exceeds min(budget.workers, capability count)`);
|
|
127
|
-
}
|
|
128
148
|
const assigned = items.flatMap((item) => Array.isArray(item.capability_ids) ? item.capability_ids : []);
|
|
129
149
|
for (const id of expected) {
|
|
130
|
-
|
|
131
|
-
if (count === 0)
|
|
132
|
-
errors.push(`${branch}: assigned capability ${id} is missing from the plan`);
|
|
133
|
-
if (count > 1)
|
|
150
|
+
if (assigned.filter((candidate) => candidate === id).length > 1) {
|
|
134
151
|
errors.push(`${branch}: assigned capability ${id} appears more than once`);
|
|
152
|
+
}
|
|
135
153
|
}
|
|
136
154
|
for (const id of assigned.filter((id) => !expected.includes(id)))
|
|
137
155
|
errors.push(`${branch}: capability ${id} was not assigned by the request`);
|
|
138
|
-
const sizes = items.map((item) => Array.isArray(item.planned_cases) ? item.planned_cases.length : 0).filter((size) => size > 0);
|
|
139
|
-
if (sizes.length > 1 && Math.max(...sizes) / Math.min(...sizes) > 1.5) {
|
|
140
|
-
errors.push(`${branch}: planned case slice ratio exceeds 1.5`);
|
|
141
|
-
}
|
|
142
156
|
}
|
|
143
157
|
return errors;
|
|
144
158
|
}
|
package/dist/proc.js
CHANGED
|
@@ -115,6 +115,21 @@ export const GRAPH_NOISE_PATTERNS = [
|
|
|
115
115
|
// Type declarations — a contract for a compiler, with no runtime behaviour.
|
|
116
116
|
'*.d.ts',
|
|
117
117
|
'*.pyi',
|
|
118
|
+
// Spec 34-6, criterion 6. Unitbob's own session reports, left in the repo root
|
|
119
|
+
// by the operator, are read as source: on a2time, 2026-08-10, two of them
|
|
120
|
+
// contributed 44 and 25 nodes and pushed their community to third-largest in
|
|
121
|
+
// the whole project. The more often you run unitbob, the dirtier its map gets —
|
|
122
|
+
// a feedback loop with no floor.
|
|
123
|
+
'/unitbob*.md',
|
|
124
|
+
// Test scaffolding and boot wiring. Factories and model specs describe the
|
|
125
|
+
// fixtures a suite builds, not a business promise a guardrail could protect;
|
|
126
|
+
// `config/initializers/` and `config/deploy/` run once at boot and at deploy.
|
|
127
|
+
// The project's own request and feature specs stay — they are evidence of what
|
|
128
|
+
// the app promises.
|
|
129
|
+
'spec/factories/',
|
|
130
|
+
'spec/models/',
|
|
131
|
+
'config/deploy/',
|
|
132
|
+
'config/initializers/',
|
|
118
133
|
];
|
|
119
134
|
export function ensureUnitbobIgnored(projectRoot) {
|
|
120
135
|
// `.graphifyignore` is unitbob's own bookkeeping, like the other two entries —
|
package/dist/runner/bootcheck.js
CHANGED
|
@@ -1,6 +1,7 @@
|
|
|
1
1
|
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
2
|
import { dirname, join } from 'node:path';
|
|
3
3
|
import { executable, runProcess } from "../proc.js";
|
|
4
|
+
import { locateRunner } from "./toolchain.js";
|
|
4
5
|
import { GUARDRAILS_DIR, HELPER_FILE } from "../files/guardrails.js";
|
|
5
6
|
import { PYTEST_INI, PYTEST_INI_FILE } from "./pytest.js";
|
|
6
7
|
import { PROVISION_TIMEOUT_MS } from "./provision.js";
|
|
@@ -86,9 +87,14 @@ async function loadRubyHelper(projectRoot, helper, deps) {
|
|
|
86
87
|
// and the global `bundle` was standing right there the whole time.
|
|
87
88
|
const localBundle = join(projectRoot, 'bin', 'bundle');
|
|
88
89
|
const command = executable(localBundle) ? localBundle : 'bundle';
|
|
90
|
+
// When Unitbob installed rspec-rails for itself, the gems this helper needs
|
|
91
|
+
// are resolved by the sidecar Gemfile, not the project's. Asking bundler
|
|
92
|
+
// without that variable would load a different set of gems than the run does,
|
|
93
|
+
// which is exactly the way a check ends up predicting the wrong thing.
|
|
94
|
+
const located = locateRunner(projectRoot, 'rspec');
|
|
89
95
|
return classify(projectRoot, 'rspec', await attempt(deps, command, ['exec', 'ruby', '-e', `require ${JSON.stringify(helper)}`], {
|
|
90
96
|
cwd: projectRoot,
|
|
91
|
-
env: { RAILS_ENV: 'test', UNITBOB_REPO_ROOT: projectRoot },
|
|
97
|
+
env: { ...located?.env, RAILS_ENV: 'test', UNITBOB_REPO_ROOT: projectRoot },
|
|
92
98
|
}),
|
|
93
99
|
// A clean load says nothing on stdout and exits 0. Anything else is the
|
|
94
100
|
// suite failing to start.
|
|
@@ -110,10 +116,25 @@ async function pytestBootCheck(projectRoot, deps) {
|
|
|
110
116
|
// first: this check must not fail because a different step was skipped.
|
|
111
117
|
mkdirSync(join(projectRoot, dirname(PYTEST_INI_FILE)), { recursive: true });
|
|
112
118
|
writeFileSync(join(projectRoot, PYTEST_INI_FILE), PYTEST_INI);
|
|
113
|
-
|
|
114
|
-
|
|
115
|
-
|
|
116
|
-
|
|
119
|
+
// The same pytest the run will use, resolved once in `locateRunner` — the
|
|
120
|
+
// sidecar under `.unitbob/` when Unitbob installed one, else the machine's own
|
|
121
|
+
// interpreter. Asking a different interpreter than the run uses is how a check
|
|
122
|
+
// ends up answering about something nobody is going to execute.
|
|
123
|
+
//
|
|
124
|
+
// When it resolves nothing we still try the two interpreters by name rather
|
|
125
|
+
// than reporting `no_runner` from a lookup. The lookup is a prediction; the
|
|
126
|
+
// spawn is the fact, and a check that stops at its own prediction can be
|
|
127
|
+
// wrong in the one direction that costs the most — refusing a project that
|
|
128
|
+
// would have answered perfectly well.
|
|
129
|
+
const located = locateRunner(projectRoot, 'pytest');
|
|
130
|
+
const candidates = located
|
|
131
|
+
? [located]
|
|
132
|
+
: [
|
|
133
|
+
{ command: 'python3', args: ['-m', 'pytest'], env: undefined },
|
|
134
|
+
{ command: 'python', args: ['-m', 'pytest'], env: undefined },
|
|
135
|
+
];
|
|
136
|
+
for (const candidate of candidates) {
|
|
137
|
+
const result = await attempt(deps, candidate.command, [...candidate.args, '-c', PYTEST_INI_FILE, '--collect-only', '-q'], { cwd: projectRoot, env: candidate.env });
|
|
117
138
|
if (result === null)
|
|
118
139
|
continue; // this interpreter is not on the machine
|
|
119
140
|
return classify(projectRoot, 'pytest', result, (proc) => pytestVerdict(proc.code));
|
|
@@ -148,7 +169,10 @@ function pytestVerdict(code) {
|
|
|
148
169
|
// turn away the majority. A file that is genuinely unparseable is caught here
|
|
149
170
|
// anyway, since `vitest list` has to parse it.
|
|
150
171
|
async function vitestBootCheck(projectRoot, deps) {
|
|
151
|
-
|
|
172
|
+
// A sidecar vitest counts as installed: it is ours, it is on disk, and it is
|
|
173
|
+
// the one the run will spawn. What stays out is `npx`, for the reason below.
|
|
174
|
+
const local = locateRunner(projectRoot, 'vitest')?.command
|
|
175
|
+
?? join(projectRoot, 'node_modules', '.bin', 'vitest');
|
|
152
176
|
// Only a vitest already installed in the project is used. Reaching for `npx`
|
|
153
177
|
// would install a package to answer a question, and installing into the
|
|
154
178
|
// user's project is not this check's business.
|
|
@@ -275,15 +299,17 @@ function hasProjectFrame(output, projectRoot, runner) {
|
|
|
275
299
|
// asked of that file. Judging the whole output at once let `.venv/lib/...`
|
|
276
300
|
// answer yes on the strength of its `lib/`, which is how a `TypeError` deep
|
|
277
301
|
// inside a dependency came back as a defect in the user's code.
|
|
278
|
-
const ownDirs = runner
|
|
279
|
-
const conventional =
|
|
302
|
+
const ownDirs = CONVENTIONAL_SOURCE_DIRS[runner] ?? [];
|
|
303
|
+
const conventional = ownDirs.length
|
|
304
|
+
? new RegExp(`(^|[\\s"'(\\[/])(${ownDirs.join('|')})/`)
|
|
305
|
+
: null;
|
|
280
306
|
for (const line of output.split('\n')) {
|
|
281
307
|
// Wherever a dependency is installed, it is not this project's code — and
|
|
282
308
|
// that has to be decided before anything below gets a chance to say yes.
|
|
283
309
|
if (INSTALLED_DEPENDENCY.test(line))
|
|
284
310
|
continue;
|
|
285
311
|
// The conventional homes of business code, relative or absolute.
|
|
286
|
-
if (conventional
|
|
312
|
+
if (conventional?.test(line))
|
|
287
313
|
return true;
|
|
288
314
|
if (line.includes(projectRoot))
|
|
289
315
|
return true;
|
|
@@ -303,6 +329,18 @@ function hasProjectFrame(output, projectRoot, runner) {
|
|
|
303
329
|
}
|
|
304
330
|
return false;
|
|
305
331
|
}
|
|
332
|
+
// Where each stack conventionally keeps its business code. Only a stack that
|
|
333
|
+
// really has such a convention gets an entry: `app/` and `lib/` are Rails, and
|
|
334
|
+
// they used to be the fallback for everything that was not vitest, which meant a
|
|
335
|
+
// Python project got Rails's layout applied to its stack traces. Python names no
|
|
336
|
+
// fixed layout at all, so it is deliberately absent — the repository-file test
|
|
337
|
+
// below is the answer there, and it is the more reliable one anyway.
|
|
338
|
+
const CONVENTIONAL_SOURCE_DIRS = {
|
|
339
|
+
rspec: ['app', 'lib'],
|
|
340
|
+
cucumber: ['app', 'lib'],
|
|
341
|
+
vitest: ['src'],
|
|
342
|
+
'cucumber-js': ['src'],
|
|
343
|
+
};
|
|
306
344
|
// Where a dependency lives once installed — never the project's own code, in
|
|
307
345
|
// any of the three languages. The last two are the languages' own installed
|
|
308
346
|
// libraries: `…/lib/ruby/3.3.0/psych.rb` is a frame the `lib/` rule below would
|
|
@@ -0,0 +1,238 @@
|
|
|
1
|
+
import { createHash } from 'node:crypto';
|
|
2
|
+
import { mkdirSync, readFileSync, renameSync, rmSync, writeFileSync } from 'node:fs';
|
|
3
|
+
import { dirname, join } from 'node:path';
|
|
4
|
+
export const RUN_STATE_FILE = 'run-state.json';
|
|
5
|
+
// A marker embedded in a reported test name or Gherkin tag. Same shape the
|
|
6
|
+
// server joins on (`CaseMarker::EMBEDDED`): a full 12-hex marker, never a prefix
|
|
7
|
+
// of a longer hex run. A case with no marker still counts as a failure — it is
|
|
8
|
+
// identified by its file and message alone.
|
|
9
|
+
const MARKER = /ubc_[0-9a-f]{12}(?![0-9a-f])/;
|
|
10
|
+
// The failures a report describes, canonically ordered, or `null` when the
|
|
11
|
+
// report cannot be read as one. `null` is not "green": a runner that died before
|
|
12
|
+
// the first test produces no set at all, and comparing against nothing would
|
|
13
|
+
// stop a branch over a harness problem the loop never even reached.
|
|
14
|
+
export function failureSet(runner, report) {
|
|
15
|
+
if (!report.trim())
|
|
16
|
+
return null;
|
|
17
|
+
const found = extract(runner, report);
|
|
18
|
+
return found && canonical(found);
|
|
19
|
+
}
|
|
20
|
+
// One hash for one set. Same set, same hash, on any machine and in any order.
|
|
21
|
+
export function digestOf(failures) {
|
|
22
|
+
return createHash('sha256').update(JSON.stringify(failures)).digest('hex');
|
|
23
|
+
}
|
|
24
|
+
export function runStatePath(projectRoot) {
|
|
25
|
+
return join(projectRoot, '.unitbob', 'suite-build', RUN_STATE_FILE);
|
|
26
|
+
}
|
|
27
|
+
// A damaged, hand-edited or absent file reads as "no previous run". Refusing to
|
|
28
|
+
// run over a bookkeeping file would turn the one soft stop in the loop into the
|
|
29
|
+
// hardest thing in it — and this file is written by a process that repair
|
|
30
|
+
// deliberately kills and restarts.
|
|
31
|
+
export function readRunState(projectRoot) {
|
|
32
|
+
try {
|
|
33
|
+
const parsed = JSON.parse(readFileSync(runStatePath(projectRoot), 'utf8'));
|
|
34
|
+
const branches = parsed?.branches;
|
|
35
|
+
if (!branches || typeof branches !== 'object' || Array.isArray(branches))
|
|
36
|
+
return {};
|
|
37
|
+
return Object.fromEntries(Object.entries(branches).filter(([, value]) => typeof value === 'string'));
|
|
38
|
+
}
|
|
39
|
+
catch {
|
|
40
|
+
return {};
|
|
41
|
+
}
|
|
42
|
+
}
|
|
43
|
+
// `undefined` forgets this branch: a run that produced no comparable set leaves
|
|
44
|
+
// the next one to be a first run again.
|
|
45
|
+
export function rememberFailures(projectRoot, branch, digest) {
|
|
46
|
+
const branches = readRunState(projectRoot);
|
|
47
|
+
if (digest === undefined)
|
|
48
|
+
delete branches[branch];
|
|
49
|
+
else
|
|
50
|
+
branches[branch] = digest;
|
|
51
|
+
const path = runStatePath(projectRoot);
|
|
52
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
53
|
+
// Written whole and moved into place. A plain write that is interrupted leaves
|
|
54
|
+
// truncated JSON, which reads back as "no previous run" — exactly at the
|
|
55
|
+
// moment a run is being killed and restarted, which is the loop this bounds.
|
|
56
|
+
const staging = `${path}.tmp`;
|
|
57
|
+
writeFileSync(staging, `${JSON.stringify({ branches }, null, 2)}\n`);
|
|
58
|
+
renameSync(staging, path);
|
|
59
|
+
}
|
|
60
|
+
export function clearRunState(projectRoot) {
|
|
61
|
+
rmSync(runStatePath(projectRoot), { force: true });
|
|
62
|
+
}
|
|
63
|
+
function canonical(failures) {
|
|
64
|
+
const byKey = new Map(failures.map((failure) => [keyOf(failure), failure]));
|
|
65
|
+
return [...byKey.keys()].sort().map((key) => byKey.get(key));
|
|
66
|
+
}
|
|
67
|
+
function keyOf(failure) {
|
|
68
|
+
return `${failure.marker}\u0000${failure.file}\u0000${failure.message}`;
|
|
69
|
+
}
|
|
70
|
+
// The connector interprets a report in exactly one other place — `boundReport`,
|
|
71
|
+
// which bounds it for transport — and this switch deliberately mirrors that
|
|
72
|
+
// one's shape rather than inventing a second dispatch. Neither of them judges a
|
|
73
|
+
// run — the server owns every verdict a report leads to. This one only asks "is
|
|
74
|
+
// this the same wall we hit last time", and its answer reaches nothing but an
|
|
75
|
+
// exit code.
|
|
76
|
+
function extract(runner, report) {
|
|
77
|
+
switch (runner) {
|
|
78
|
+
case 'rspec':
|
|
79
|
+
return fromRspec(report);
|
|
80
|
+
case 'vitest':
|
|
81
|
+
return fromVitest(report);
|
|
82
|
+
case 'pytest':
|
|
83
|
+
return fromJunitXml(report);
|
|
84
|
+
case 'cucumber':
|
|
85
|
+
case 'cucumber-js':
|
|
86
|
+
return fromCucumberMessages(report);
|
|
87
|
+
case 'pytest-bdd':
|
|
88
|
+
return fromPytestBdd(report);
|
|
89
|
+
default:
|
|
90
|
+
return null;
|
|
91
|
+
}
|
|
92
|
+
}
|
|
93
|
+
function fromRspec(report) {
|
|
94
|
+
const data = parseObject(report);
|
|
95
|
+
if (!Array.isArray(data?.examples))
|
|
96
|
+
return null;
|
|
97
|
+
return rows(data.examples).flatMap((example) => {
|
|
98
|
+
if (example.status === 'passed')
|
|
99
|
+
return [];
|
|
100
|
+
const name = `${text(example.description)} ${text(example.full_description)}`;
|
|
101
|
+
const exception = example.exception;
|
|
102
|
+
return [failure(name, text(example.file_path), text(exception?.message))];
|
|
103
|
+
});
|
|
104
|
+
}
|
|
105
|
+
function fromVitest(report) {
|
|
106
|
+
const data = parseObject(report);
|
|
107
|
+
if (!Array.isArray(data?.testResults))
|
|
108
|
+
return null;
|
|
109
|
+
return rows(data.testResults).flatMap((file) => {
|
|
110
|
+
const assertions = Array.isArray(file.assertionResults) ? rows(file.assertionResults) : [];
|
|
111
|
+
return assertions.flatMap((assertion) => {
|
|
112
|
+
if (assertion.status === 'passed')
|
|
113
|
+
return [];
|
|
114
|
+
const messages = Array.isArray(assertion.failureMessages) ? assertion.failureMessages : [];
|
|
115
|
+
const name = `${text(assertion.title)} ${text(assertion.fullName)}`;
|
|
116
|
+
return [failure(name, text(file.name), messages.map((m) => text(m)).join('\n'))];
|
|
117
|
+
});
|
|
118
|
+
});
|
|
119
|
+
}
|
|
120
|
+
// pytest's JUnit XML, read the way `boundReport` already reads it: by pattern,
|
|
121
|
+
// because a whole XML parser to answer one yes/no question is a dependency this
|
|
122
|
+
// package does not need.
|
|
123
|
+
//
|
|
124
|
+
// `failure` and `error`, but not `skipped` — and this is the one place the set
|
|
125
|
+
// deliberately does not match the server's "did not pass". A skip never moves:
|
|
126
|
+
// it reports the same thing on every run forever, so counting it here would stop
|
|
127
|
+
// a branch whose only permanent case is a skip, and it would do it while the
|
|
128
|
+
// repair was still fixing everything else. A skip that should not be there is
|
|
129
|
+
// caught at upload, where it costs a message rather than a branch.
|
|
130
|
+
function fromJunitXml(report) {
|
|
131
|
+
if (!/<testsuites?\b/.test(report))
|
|
132
|
+
return null;
|
|
133
|
+
const cases = report.match(/<testcase\b[^>]*(?:\/>|>[\s\S]*?<\/testcase>)/g) ?? [];
|
|
134
|
+
return cases.flatMap((testcase) => {
|
|
135
|
+
const problem = testcase.match(/<(?:failure|error)\b[^>]*(?:\/>|>[\s\S]*?<\/(?:failure|error)>)/);
|
|
136
|
+
if (!problem)
|
|
137
|
+
return [];
|
|
138
|
+
const name = attribute(testcase, 'name');
|
|
139
|
+
const file = attribute(testcase, 'file') || attribute(testcase, 'classname');
|
|
140
|
+
return [failure(name, file, attribute(problem[0], 'message'))];
|
|
141
|
+
});
|
|
142
|
+
}
|
|
143
|
+
// Cucumber Messages (NDJSON), both the Ruby and the JS emitter. One scenario is
|
|
144
|
+
// spread over several envelopes: the pickle holds its name, tags and file, the
|
|
145
|
+
// testCase maps its steps, and testStepFinished carries each step's result.
|
|
146
|
+
function fromCucumberMessages(report) {
|
|
147
|
+
const envelopes = [];
|
|
148
|
+
for (const line of report.split('\n')) {
|
|
149
|
+
if (!line.trim())
|
|
150
|
+
continue;
|
|
151
|
+
const parsed = parseObject(line);
|
|
152
|
+
if (!parsed)
|
|
153
|
+
return null;
|
|
154
|
+
envelopes.push(parsed);
|
|
155
|
+
}
|
|
156
|
+
if (envelopes.length === 0)
|
|
157
|
+
return null;
|
|
158
|
+
const pickles = indexBy(envelopes, 'pickle');
|
|
159
|
+
const testCases = indexBy(envelopes, 'testCase');
|
|
160
|
+
const results = new Map();
|
|
161
|
+
for (const envelope of envelopes) {
|
|
162
|
+
const finished = envelope.testStepFinished;
|
|
163
|
+
if (!finished)
|
|
164
|
+
continue;
|
|
165
|
+
const startedId = text(finished.testCaseStartedId);
|
|
166
|
+
const list = results.get(startedId) ?? [];
|
|
167
|
+
list.push(finished.testStepResult ?? {});
|
|
168
|
+
results.set(startedId, list);
|
|
169
|
+
}
|
|
170
|
+
return envelopes.flatMap((envelope) => {
|
|
171
|
+
const started = envelope.testCaseStarted;
|
|
172
|
+
if (!started)
|
|
173
|
+
return [];
|
|
174
|
+
const testCase = testCases.get(text(started.testCaseId)) ?? {};
|
|
175
|
+
const pickle = pickles.get(text(testCase.pickleId)) ?? {};
|
|
176
|
+
const steps = results.get(text(started.id)) ?? [];
|
|
177
|
+
const failed = steps.filter((step) => text(step.status) !== 'PASSED' && text(step.status) !== 'SKIPPED');
|
|
178
|
+
if (failed.length === 0)
|
|
179
|
+
return [];
|
|
180
|
+
const tags = Array.isArray(pickle.tags) ? pickle.tags : [];
|
|
181
|
+
const tagText = rows(tags).map((tag) => text(tag.name)).join(' ');
|
|
182
|
+
const message = failed.map((step) => text(step.message)).find((line) => line.trim()) ?? '';
|
|
183
|
+
return [failure(`${tagText} ${text(pickle.name)}`, text(pickle.uri), message)];
|
|
184
|
+
});
|
|
185
|
+
}
|
|
186
|
+
// The connector's own pytest-bdd report (`runner/pytestBddPlugin.ts`). It names
|
|
187
|
+
// no file — the whole behavioral bundle is one run — so the scenario's marker
|
|
188
|
+
// and message carry the identity alone.
|
|
189
|
+
function fromPytestBdd(report) {
|
|
190
|
+
const data = parseObject(report);
|
|
191
|
+
if (!Array.isArray(data?.scenarios))
|
|
192
|
+
return null;
|
|
193
|
+
return rows(data.scenarios).flatMap((scenario) => {
|
|
194
|
+
if (text(scenario.status) === 'passed')
|
|
195
|
+
return [];
|
|
196
|
+
const tags = Array.isArray(scenario.tags) ? scenario.tags.map((tag) => text(tag)).join(' ') : '';
|
|
197
|
+
return [failure(`${tags} ${text(scenario.name)}`, '', text(scenario.failure))];
|
|
198
|
+
});
|
|
199
|
+
}
|
|
200
|
+
// Only the first line of a message. Later lines are backtraces and diffs, which
|
|
201
|
+
// carry object ids and absolute paths that differ between two runs of the same
|
|
202
|
+
// unchanged failure — the very drift that would make this comparison useless.
|
|
203
|
+
function failure(name, file, message) {
|
|
204
|
+
return {
|
|
205
|
+
marker: name.match(MARKER)?.[0] ?? '',
|
|
206
|
+
file,
|
|
207
|
+
message: message.split('\n')[0]?.trim() ?? '',
|
|
208
|
+
};
|
|
209
|
+
}
|
|
210
|
+
function indexBy(envelopes, key) {
|
|
211
|
+
const found = new Map();
|
|
212
|
+
for (const envelope of envelopes) {
|
|
213
|
+
const item = envelope[key];
|
|
214
|
+
if (item)
|
|
215
|
+
found.set(text(item.id), item);
|
|
216
|
+
}
|
|
217
|
+
return found;
|
|
218
|
+
}
|
|
219
|
+
function attribute(tag, name) {
|
|
220
|
+
return tag.match(new RegExp(`\\b${name}="([^"]*)"`))?.[1] ?? '';
|
|
221
|
+
}
|
|
222
|
+
function rows(values) {
|
|
223
|
+
return values.filter((value) => value !== null && typeof value === 'object' && !Array.isArray(value));
|
|
224
|
+
}
|
|
225
|
+
function text(value) {
|
|
226
|
+
return typeof value === 'string' ? value : '';
|
|
227
|
+
}
|
|
228
|
+
function parseObject(source) {
|
|
229
|
+
try {
|
|
230
|
+
const parsed = JSON.parse(source);
|
|
231
|
+
return parsed && typeof parsed === 'object' && !Array.isArray(parsed)
|
|
232
|
+
? parsed
|
|
233
|
+
: null;
|
|
234
|
+
}
|
|
235
|
+
catch {
|
|
236
|
+
return null;
|
|
237
|
+
}
|
|
238
|
+
}
|