testguard-cli 0.16.0 → 0.17.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +51 -0
- package/README.md +152 -13
- package/package.json +1 -1
- package/spec/GATE-SEMANTICS.md +295 -0
- package/spec/README.md +27 -2
- package/spec/lib/authoring-input.mjs +40 -0
- package/spec/lib/origins.mjs +56 -0
- package/spec/lib/validate.mjs +120 -1
- package/spec/schemas/annotations.schema.json +65 -0
- package/spec/schemas/authoring-input.schema.json +67 -0
- package/spec/schemas/brief.schema.json +2 -0
- package/spec/schemas/claims.schema.json +2 -8
- package/spec/schemas/common.schema.json +73 -0
- package/spec/schemas/evidence.schema.json +37 -8
- package/spec/schemas/status.schema.json +6 -2
- package/src/brief/brief.mjs +11 -4
- package/src/claims/annotation-placement.mjs +357 -0
- package/src/claims/annotations.mjs +61 -2
- package/src/cli.mjs +40 -5
- package/src/commands/admit.mjs +13 -0
- package/src/commands/annotations.mjs +67 -0
- package/src/commands/claims.mjs +8 -4
- package/src/commands/probe.mjs +50 -7
- package/src/commands/replay.mjs +7 -0
- package/src/commands/scaffold.mjs +49 -2
- package/src/commands/sweep.mjs +7 -0
- package/src/evidence/writer.mjs +12 -4
- package/src/git.mjs +4 -4
- package/src/init/templates/SKILL.md +35 -0
- package/src/probe/attribution.mjs +10 -0
- package/src/probe/contention.mjs +20 -2
- package/src/probe/cost.mjs +6 -1
- package/src/probe/inject.mjs +2 -6
- package/src/probe/origin-policy.mjs +69 -0
- package/src/probe/policy-freshness.mjs +74 -0
- package/src/probe/probe.mjs +48 -23
- package/src/probe/runners/discovery.mjs +2 -0
- package/src/probe/runners/jest.mjs +2 -2
- package/src/probe/runners/lifecycle.mjs +58 -0
- package/src/probe/runners/playwright.mjs +6 -2
- package/src/probe/runners/python.mjs +10 -4
- package/src/probe/runners/shared.mjs +12 -6
- package/src/probe/runners/vitest.mjs +3 -3
- package/src/probe/universe.mjs +47 -0
- package/src/probe/worktree.mjs +3 -1
- package/src/render.mjs +17 -1
- package/src/replay/replay.mjs +8 -4
- package/src/scaffold/admission.mjs +117 -0
- package/src/scaffold/append.mjs +41 -0
- package/src/scaffold/apply-append.mjs +71 -0
- package/src/scaffold/fix-input.mjs +119 -0
- package/src/scaffold/fix-paths.mjs +90 -0
- package/src/scaffold/intent.mjs +2 -0
- package/src/scaffold/open-append.mjs +44 -0
- package/src/scaffold/plan-append.mjs +37 -0
- package/src/scaffold/plan-intent-input.mjs +38 -0
- package/src/scaffold/prepare-append.mjs +46 -0
- package/src/scaffold/request.mjs +30 -0
- package/src/scaffold/scaffold.mjs +23 -10
- package/src/status/status.mjs +34 -9
- package/src/sweep/pool.mjs +33 -0
- package/src/sweep/sweep.mjs +9 -4
|
@@ -54,6 +54,21 @@ sections already listing this project.
|
|
|
54
54
|
|
|
55
55
|
## The one source of truth
|
|
56
56
|
|
|
57
|
+
Missing source-link advisories in claims/status never replace `next`, change
|
|
58
|
+
verdicts or prove intent. Exact anchors remain checked without annotations.
|
|
59
|
+
Origin summaries are likewise informational: status counts current declared
|
|
60
|
+
claims; evidence/briefs count recorded declarations and records, including
|
|
61
|
+
mixed sources. Source kinds are not authenticated independence. Never use an
|
|
62
|
+
origin label to waive a survivor.
|
|
63
|
+
Optional source links: `testguard claims --annotate --claim <ID>` is a read-only
|
|
64
|
+
file-header preview. Only with explicit permission to edit source, rerun with
|
|
65
|
+
`--apply`; each invocation recomputes its plan. Review refusals and partial
|
|
66
|
+
failures, and retain the reported private recovery originals. Never automatically
|
|
67
|
+
remove an existing lock or roll back over user edits. Placement is file-level,
|
|
68
|
+
not symbol ownership, verified behavior or authenticated intent. Missing links
|
|
69
|
+
do not disable exact fault-anchor checks. Authoring never replaces the status
|
|
70
|
+
operating loop or writes canonical claims/evidence/baselines.
|
|
71
|
+
|
|
57
72
|
```bash
|
|
58
73
|
testguard status --json
|
|
59
74
|
```
|
|
@@ -76,6 +91,18 @@ Exit codes: `0` clean · `1` unproven claims (or drift, or unclaimed changes) ·
|
|
|
76
91
|
|
|
77
92
|
## Every change needs a claim
|
|
78
93
|
|
|
94
|
+
Scaffold proposes mechanical faults, not intended behavior. New drafts declare
|
|
95
|
+
`source.kind: inferred` even when grouped by an annotation. Before adopting a
|
|
96
|
+
draft, ask for the intended observable behavior from a requirement, ADR, bug or
|
|
97
|
+
incident independently of the implementation: inputs, expected outcome and
|
|
98
|
+
forbidden outcome. Record the supplied source kind/ref; when unavailable, keep
|
|
99
|
+
inferred and say the intent remains unestablished. Do not promote origin because
|
|
100
|
+
an annotation exists or tests pass. References are opaque, not authenticated;
|
|
101
|
+
verification commands never fetch them. Preserve existing supplied metadata.
|
|
102
|
+
Review each proposal against that intent, then probe. Do not paraphrase the code
|
|
103
|
+
and call that independent intent, or automatically move drafts into canonical
|
|
104
|
+
claims.
|
|
105
|
+
|
|
79
106
|
`probe` verifies only the claims that exist; it is silent about unclaimed
|
|
80
107
|
code by construction. So, for every source file you create or change:
|
|
81
108
|
|
|
@@ -108,6 +135,14 @@ code by construction. So, for every source file you create or change:
|
|
|
108
135
|
|
|
109
136
|
## The fix loop
|
|
110
137
|
|
|
138
|
+
For independently supplied intent in an existing disposable draft, use
|
|
139
|
+
`testguard scaffold <source...> --into <draft.json> --claim <existing-ID> --json`
|
|
140
|
+
to preview appended faults without writes. Omit `--json` only when explicitly
|
|
141
|
+
authorized to update that draft; private recovery is retained. Never use a
|
|
142
|
+
canonical claims/evidence/baseline path. Preserve intended behavior and declared
|
|
143
|
+
origin; generated faults are not independent intent or verification. Review the
|
|
144
|
+
faults and probe the selected draft before promoting it to canonical claims.
|
|
145
|
+
|
|
111
146
|
1. `testguard status --json` → take `next.target`.
|
|
112
147
|
2. Write the test. It must **fail when the fault is applied and pass on HEAD**. To check the first half by hand: apply `find` → `replace` in the target file, run the defender, restore.
|
|
113
148
|
3. `testguard admit <test-file> --claim <ID>` — the two gates as one verb, on your uncommitted test, without touching the tree: `ADMITTED` (exit 0) means the test is green on HEAD and fails on every fault of the claim, 3/3; `NOT ADMITTED` (exit 1) names the first blocking fault and what to do. `--fault <FID>` judges one fault; `--confirm 1` gives a fast **provisional** `ADMITTED?` that must be confirmed at 3 before you commit. (`testguard probe --claim <ID> --include-dirty` is the same run with the full report.)
|
|
@@ -11,8 +11,17 @@
|
|
|
11
11
|
* worktree is. Everything it needs is passed in.
|
|
12
12
|
*/
|
|
13
13
|
|
|
14
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
15
|
+
|
|
14
16
|
const isKill = (r) => r.outcome === 'fail' && (r.assertionFailures ?? 0) > 0;
|
|
15
17
|
|
|
18
|
+
/** Compare only the claim metadata actually recorded in evidence. */
|
|
19
|
+
export function sameClaimMetadata(recorded, current) {
|
|
20
|
+
if (!recorded || !current) return false;
|
|
21
|
+
const metadata = ({ id, statement, severity, source, producedBy }) => ({ id, statement, severity, source, producedBy });
|
|
22
|
+
return isDeepStrictEqual(metadata(recorded), metadata(current));
|
|
23
|
+
}
|
|
24
|
+
|
|
16
25
|
/**
|
|
17
26
|
* Fold one escalation run into the running attribution state.
|
|
18
27
|
*
|
|
@@ -118,6 +127,7 @@ export function defenderSelection(claim, fault) {
|
|
|
118
127
|
|
|
119
128
|
export function isReusable(prior, current) {
|
|
120
129
|
if (!prior || !prior.subject?.contentHash) return false;
|
|
130
|
+
if (current.claim && !sameClaimMetadata(prior.claim, current.claim)) return false;
|
|
121
131
|
// A record that exists because the probe threw is a statement about the run,
|
|
122
132
|
// not about the code. Reusing it would make one disturbed run permanent.
|
|
123
133
|
if (prior.detail?.reason === 'probe-error') return false;
|
package/src/probe/contention.mjs
CHANGED
|
@@ -37,9 +37,13 @@ export function processList({ platform = process.platform, run = spawnSync } = {
|
|
|
37
37
|
try {
|
|
38
38
|
const r = platform === 'win32'
|
|
39
39
|
? run('wmic', ['process', 'get', 'ProcessId,CommandLine', '/format:csv'], { encoding: 'utf8', timeout: 5000 })
|
|
40
|
-
: run('ps', ['-Ao', 'pid=,args='], { encoding: 'utf8', timeout: 5000 });
|
|
40
|
+
: run('ps', ['-Ao', 'pid=,ppid=,stat=,args='], { encoding: 'utf8', timeout: 5000 });
|
|
41
41
|
if (r.status !== 0 || !r.stdout) return [];
|
|
42
42
|
return r.stdout.split('\n').map((line) => {
|
|
43
|
+
if (platform !== 'win32') {
|
|
44
|
+
const row = /^\s*(\d+)\s+(\d+)\s+(\S+)\s+(.*)$/.exec(line);
|
|
45
|
+
if (row) return { pid: Number(row[1]), ppid: Number(row[2]), stat: row[3], command: row[4].trim() };
|
|
46
|
+
}
|
|
43
47
|
const m = platform === 'win32' ? /^[^,]*,(.*),(\d+)\s*$/.exec(line) : /^\s*(\d+)\s+(.*)$/.exec(line);
|
|
44
48
|
if (!m) return null;
|
|
45
49
|
const [pid, command] = platform === 'win32' ? [m[2], m[1]] : [m[1], m[2]];
|
|
@@ -57,8 +61,22 @@ export function processList({ platform = process.platform, run = spawnSync } = {
|
|
|
57
61
|
*/
|
|
58
62
|
export function detectContention({ self = process.pid, alive = defaultAlive, ...opts } = {}) {
|
|
59
63
|
const rows = processList(opts);
|
|
64
|
+
const owned = new Set([self]);
|
|
65
|
+
let changed = true;
|
|
66
|
+
while (changed) {
|
|
67
|
+
changed = false;
|
|
68
|
+
for (const row of rows) {
|
|
69
|
+
if (owned.has(row.ppid) && !owned.has(row.pid)) { owned.add(row.pid); changed = true; }
|
|
70
|
+
}
|
|
71
|
+
}
|
|
72
|
+
const ancestors = new Set();
|
|
73
|
+
let parent = rows.find((row) => row.pid === self)?.ppid;
|
|
74
|
+
while (parent && !ancestors.has(parent)) {
|
|
75
|
+
ancestors.add(parent);
|
|
76
|
+
parent = rows.find((row) => row.pid === parent)?.ppid;
|
|
77
|
+
}
|
|
60
78
|
const runners = rows
|
|
61
|
-
.filter((p) => p.pid
|
|
79
|
+
.filter((p) => !owned.has(p.pid) && !ancestors.has(p.pid) && !p.stat?.startsWith('Z') && RUNNER_RE.test(p.command) && (p.ppid !== undefined || !SELF_RE.test(p.command)))
|
|
62
80
|
// `ps` lists the grep/ps itself and any shell wrapper; a runner's command
|
|
63
81
|
// line always names its own binary, so require it to look like an exec.
|
|
64
82
|
.filter((p) => !/^\s*(ps|grep|wmic)\b/.test(p.command))
|
package/src/probe/cost.mjs
CHANGED
|
@@ -92,10 +92,11 @@ export function defenderCosts(records) {
|
|
|
92
92
|
* turns a twenty-four-minute gate into a one-minute one, and until a report
|
|
93
93
|
* names the file nobody knows which test to split.
|
|
94
94
|
*/
|
|
95
|
-
export function costReport(records) {
|
|
95
|
+
export function costReport(records, { run } = {}) {
|
|
96
96
|
const claims = claimCosts(records);
|
|
97
97
|
const defenders = defenderCosts(records);
|
|
98
98
|
return {
|
|
99
|
+
...(run?.measurements ? { measurements: { ...run.measurements } } : {}),
|
|
99
100
|
totalMs: claims.reduce((a, c) => a + c.ms, 0),
|
|
100
101
|
totalRuns: claims.reduce((a, c) => a + c.runs, 0),
|
|
101
102
|
reusedRecords: records.filter((r) => r.reusedFrom).length,
|
|
@@ -118,6 +119,10 @@ export const formatMs = secs;
|
|
|
118
119
|
*/
|
|
119
120
|
export function renderCost(report, { limit = 10 } = {}) {
|
|
120
121
|
const out = [];
|
|
122
|
+
if (report.measurements) {
|
|
123
|
+
const m = report.measurements;
|
|
124
|
+
out.push(`actual elapsed ${secs(m.elapsedMs)}; fresh runners ${secs(m.runnerMs)} across ${m.runnerInvocations} invocations; setup/discovery/other overhead ${secs(m.overheadMs)}`);
|
|
125
|
+
}
|
|
121
126
|
if (!report.records) return 'no evidence records — run `testguard probe` first; cost is read back from the runs it records.';
|
|
122
127
|
out.push(`${report.records} fault record${report.records === 1 ? '' : 's'} cost ${secs(report.totalMs)} across ${report.totalRuns} defender runs${report.reusedRecords ? ` (${report.reusedRecords} reused; their runs are what they cost when reuse does not apply)` : ''}.`);
|
|
123
128
|
out.push('');
|
package/src/probe/inject.mjs
CHANGED
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { registerCleanup } from './runners/lifecycle.mjs';
|
|
1
2
|
import { readFileSync, writeFileSync } from 'node:fs';
|
|
2
3
|
import { join } from 'node:path';
|
|
3
4
|
|
|
@@ -19,12 +20,7 @@ function restoreAll() {
|
|
|
19
20
|
function installHandlers() {
|
|
20
21
|
if (handlersInstalled) return;
|
|
21
22
|
handlersInstalled = true;
|
|
22
|
-
|
|
23
|
-
process.on(sig, () => {
|
|
24
|
-
restoreAll();
|
|
25
|
-
process.exit(130);
|
|
26
|
-
});
|
|
27
|
-
}
|
|
23
|
+
registerCleanup(restoreAll, { priority: 1 });
|
|
28
24
|
process.on('uncaughtException', (err) => {
|
|
29
25
|
restoreAll();
|
|
30
26
|
throw err;
|
|
@@ -0,0 +1,69 @@
|
|
|
1
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
2
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
3
|
+
import { CLAIM_SOURCE_KINDS } from '../../spec/lib/origins.mjs';
|
|
4
|
+
import { sameClaimMetadata, subjectOf } from './attribution.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
|
|
7
|
+
/** Admit the planned probe option before loading claims or executing/writing. */
|
|
8
|
+
export function originPolicyRequest(values, command = 'probe') {
|
|
9
|
+
const raw = values['require-origin'];
|
|
10
|
+
if (raw === undefined) return undefined;
|
|
11
|
+
if (command !== 'probe') throw new TypeError('--require-origin is only valid on probe');
|
|
12
|
+
if (!Array.isArray(raw) || !raw.length || raw.some((v) => typeof v !== 'string')) {
|
|
13
|
+
throw new TypeError('--require-origin must name declared source kinds');
|
|
14
|
+
}
|
|
15
|
+
const kinds = raw.flatMap((v) => v.split(',').map((k) => k.trim()));
|
|
16
|
+
if (kinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) throw new TypeError('--require-origin contains an empty or unsupported kind');
|
|
17
|
+
if (values.claim !== undefined || values['allow-empty'] || values.ref !== undefined || values['ignore-dirty']) {
|
|
18
|
+
throw new TypeError('--require-origin needs a complete current-project run without --claim, --allow-empty, --ref or --ignore-dirty');
|
|
19
|
+
}
|
|
20
|
+
const n = Number(values.confirm ?? 3);
|
|
21
|
+
if (!Number.isInteger(n) || n < 3) throw new TypeError('--require-origin requires --confirm of at least 3');
|
|
22
|
+
return [...new Set(kinds)].sort();
|
|
23
|
+
}
|
|
24
|
+
|
|
25
|
+
/** Internal policy only: fresh must come from the existing input-binding checker. */
|
|
26
|
+
export function evaluateOriginPolicy({ claims, evidence, eligibleKinds, fresh }) {
|
|
27
|
+
if (eligibleKinds === undefined) return undefined;
|
|
28
|
+
if (!Array.isArray(eligibleKinds) || !eligibleKinds.length || eligibleKinds.some((k) => !CLAIM_SOURCE_KINDS.includes(k))) {
|
|
29
|
+
throw new TypeError('origin policy needs a nonempty set of declared source kinds');
|
|
30
|
+
}
|
|
31
|
+
const eligible = [...new Set(eligibleKinds)].sort();
|
|
32
|
+
const reasons = new Set();
|
|
33
|
+
const claimsValid = validate('claims', claims).ok;
|
|
34
|
+
const evidenceValid = validate('evidence', evidence).ok;
|
|
35
|
+
if (!claimsValid) reasons.add('invalid-claims');
|
|
36
|
+
if (!evidenceValid) reasons.add('invalid-evidence');
|
|
37
|
+
if (fresh !== true) reasons.add('stale-inputs');
|
|
38
|
+
const current = claimsValid ? claims.claims : [];
|
|
39
|
+
const records = evidenceValid ? evidence.records : [];
|
|
40
|
+
const key = (claimId, faultId) => JSON.stringify([claimId, faultId]);
|
|
41
|
+
const expected = new Map(current.flatMap((claim) => claim.faults.map((fault) => [key(claim.id, fault.id), { claim, fault }])));
|
|
42
|
+
if (!current.length || !expected.size) reasons.add('empty-universe');
|
|
43
|
+
if (evidenceValid && (evidence.run.provisional === true || evidence.run.confirmRuns < 3)) reasons.add('provisional-evidence');
|
|
44
|
+
if (evidenceValid && methodOf(evidence.run) !== 'fault-injection') reasons.add('incompatible-method');
|
|
45
|
+
if (current.some((c) => c.faults.some((f) => methodOf(f) !== 'fault-injection'))) reasons.add('incompatible-method');
|
|
46
|
+
const seen = new Set();
|
|
47
|
+
for (const record of records) {
|
|
48
|
+
const identity = key(record.claim.id, record.subject.id);
|
|
49
|
+
if (seen.has(identity)) reasons.add('duplicate-record');
|
|
50
|
+
seen.add(identity);
|
|
51
|
+
const pair = expected.get(identity);
|
|
52
|
+
if (!pair) { reasons.add('extra-record'); continue; }
|
|
53
|
+
if (!sameClaimMetadata(record.claim, pair.claim)) reasons.add('claim-metadata-mismatch');
|
|
54
|
+
if (methodOf(pair.fault) !== 'fault-injection') continue;
|
|
55
|
+
const subject = subjectOf(pair.fault, sha256);
|
|
56
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash'].some((field) => record.subject[field] !== subject[field])
|
|
57
|
+
|| !isDeepStrictEqual(record.subject.producedBy, subject.producedBy)) reasons.add('fault-mismatch');
|
|
58
|
+
}
|
|
59
|
+
if ([...expected.keys()].some((identity) => !seen.has(identity))) reasons.add('missing-record');
|
|
60
|
+
const ineligibleClaims = current.filter((c) => !eligible.includes(c.source.kind)).map((c) => c.id).sort();
|
|
61
|
+
const nonKilledFaults = [...new Map(records.filter((r) => r.verdict !== 'killed').map((r) =>
|
|
62
|
+
[key(r.claim.id, r.subject.id), { claimId: r.claim.id, faultId: r.subject.id }])).values()]
|
|
63
|
+
.sort((a, b) => a.claimId.localeCompare(b.claimId) || a.faultId.localeCompare(b.faultId));
|
|
64
|
+
return {
|
|
65
|
+
state: reasons.size ? 'unavailable' : ineligibleClaims.length || nonKilledFaults.length ? 'failed' : 'passed',
|
|
66
|
+
eligibleKinds: eligible, claims: current.length, faults: expected.size,
|
|
67
|
+
ineligibleClaims, nonKilledFaults, unavailableReasons: [...reasons].sort(),
|
|
68
|
+
};
|
|
69
|
+
}
|
|
@@ -0,0 +1,74 @@
|
|
|
1
|
+
import { closeSync, constants, fstatSync, openSync, readFileSync, realpathSync } from 'node:fs';
|
|
2
|
+
import { isAbsolute, join, relative, sep } from 'node:path';
|
|
3
|
+
import { isDeepStrictEqual } from 'node:util';
|
|
4
|
+
import { validate, methodOf } from '../../spec/lib/validate.mjs';
|
|
5
|
+
import { sha256 } from '../util/hash.mjs';
|
|
6
|
+
import { defenderSelection, isReusable, subjectOf } from './attribution.mjs';
|
|
7
|
+
import { discoverDefendersDetailed } from './discover.mjs';
|
|
8
|
+
import { resolveDefenders } from './runners/shared.mjs';
|
|
9
|
+
import { hashNativeTestUniverse } from './universe.mjs';
|
|
10
|
+
|
|
11
|
+
// Point-in-time reads, not a transaction over the whole project. Never turn a
|
|
12
|
+
// missing file into the empty-file hash that diagnostic probe records can use.
|
|
13
|
+
function currentHash(projectDir, file) {
|
|
14
|
+
if (typeof file !== 'string' || isAbsolute(file) || file.split(/[\\/]/).includes('..')) throw new Error('unsafe input path');
|
|
15
|
+
const path = join(projectDir, file);
|
|
16
|
+
const resolved = realpathSync(path);
|
|
17
|
+
const inside = relative(realpathSync(projectDir), resolved);
|
|
18
|
+
if (isAbsolute(inside) || inside === '..' || inside.startsWith(`..${sep}`)) throw new Error('input escaped project');
|
|
19
|
+
const fd = openSync(path, constants.O_RDONLY | constants.O_NOFOLLOW | constants.O_NONBLOCK);
|
|
20
|
+
try {
|
|
21
|
+
const before = fstatSync(fd);
|
|
22
|
+
if (!before.isFile()) throw new Error('input is not a regular file');
|
|
23
|
+
const hash = sha256(readFileSync(fd));
|
|
24
|
+
const after = fstatSync(fd);
|
|
25
|
+
if (before.size !== after.size || before.mtimeMs !== after.mtimeMs || before.ctimeMs !== after.ctimeMs || realpathSync(path) !== resolved) throw new Error('input changed during read');
|
|
26
|
+
return hash;
|
|
27
|
+
} finally { closeSync(fd); }
|
|
28
|
+
}
|
|
29
|
+
|
|
30
|
+
/** Internal policy boundary: native listing plus current per-fault bindings.
|
|
31
|
+
* Legacy/static universes and any uncertainty refuse; verdicts remain intact.
|
|
32
|
+
*/
|
|
33
|
+
export function originPolicyInputsFresh({ projectDir, claims, evidence, universe }) {
|
|
34
|
+
try {
|
|
35
|
+
if (!validate('claims', claims).ok || !validate('evidence', evidence).ok) return false;
|
|
36
|
+
if (methodOf(evidence.run) !== 'fault-injection' || evidence.run.provisional || evidence.run.confirmRuns < 3) return false;
|
|
37
|
+
const testUniverseHash = hashNativeTestUniverse(universe.manifests);
|
|
38
|
+
if (universe.testUniverseHash !== testUniverseHash) return false;
|
|
39
|
+
const matchesRunner = (recorded, current) => recorded?.name === current?.name && recorded?.version === current?.version;
|
|
40
|
+
if (!universe.primaryRunner || !matchesRunner(evidence.run.runner, universe.primaryRunner)) return false;
|
|
41
|
+
if ((evidence.run.runners ?? []).some((recorded) => !universe.runners?.some((current) => matchesRunner(recorded, current)))) return false;
|
|
42
|
+
const allTests = Object.freeze([...new Set([...universe.manifests.values()].flatMap((m) => m.files))].sort());
|
|
43
|
+
const expected = new Map();
|
|
44
|
+
for (const claim of claims.claims) for (const fault of claim.faults) {
|
|
45
|
+
if (methodOf(fault) !== 'fault-injection') return false;
|
|
46
|
+
expected.set(JSON.stringify([claim.id, fault.id]), { claim, fault });
|
|
47
|
+
}
|
|
48
|
+
if (!expected.size || evidence.records.length !== expected.size) return false;
|
|
49
|
+
const seen = new Set();
|
|
50
|
+
for (const record of evidence.records) {
|
|
51
|
+
const key = JSON.stringify([record.claim.id, record.subject.id]);
|
|
52
|
+
const current = expected.get(key);
|
|
53
|
+
if (!current || seen.has(key)) return false;
|
|
54
|
+
seen.add(key);
|
|
55
|
+
const { claim, fault } = current;
|
|
56
|
+
const subject = subjectOf(fault, sha256);
|
|
57
|
+
if (['kind', 'id', 'file', 'faultClass', 'contentHash', 'producedBy'].some((field) => !isDeepStrictEqual(record.subject[field], subject[field]))) return false;
|
|
58
|
+
const { requested, selectionSource } = defenderSelection(claim, fault);
|
|
59
|
+
const discovery = requested.length ? null : discoverDefendersDetailed(projectDir, fault.file, allTests);
|
|
60
|
+
if (discovery?.indeterminate.length) return false;
|
|
61
|
+
const resolved = requested.length ? resolveDefenders(projectDir, requested) : discovery.canDetect;
|
|
62
|
+
if (resolved.some((file) => !allTests.includes(file))) return false;
|
|
63
|
+
const hashes = (files) => Object.fromEntries(files.map((file) => [file, currentHash(projectDir, file)]));
|
|
64
|
+
const inputs = {
|
|
65
|
+
targetHash: currentHash(projectDir, fault.file),
|
|
66
|
+
defenderHashes: hashes(resolved),
|
|
67
|
+
testUniverseHash,
|
|
68
|
+
discoveryHashes: hashes(discovery?.dependencies ?? []),
|
|
69
|
+
};
|
|
70
|
+
if (!isReusable(record, { claim, requested, resolved, selectionSource, contentHash: subject.contentHash, inputs })) return false;
|
|
71
|
+
}
|
|
72
|
+
return true;
|
|
73
|
+
} catch { return false; }
|
|
74
|
+
}
|
package/src/probe/probe.mjs
CHANGED
|
@@ -1,3 +1,6 @@
|
|
|
1
|
+
import { resetInterpreterCache } from './runners/python.mjs';
|
|
2
|
+
import { assertNotCancelled } from './runners/lifecycle.mjs';
|
|
3
|
+
import { performance } from 'node:perf_hooks';
|
|
1
4
|
import { existsSync, readFileSync, realpathSync as fsRealpathSync } from 'node:fs';
|
|
2
5
|
import { gitRaw } from '../git.mjs';
|
|
3
6
|
import { createRequire } from 'node:module';
|
|
@@ -16,7 +19,9 @@ import { classifyIndependence } from './independence.mjs';
|
|
|
16
19
|
import { escalationStart, foldEscalationRun, escalationResult, flakeRate, killersFromRuns, subjectOf, isReusable, defenderSelection } from './attribution.mjs';
|
|
17
20
|
import { hashFile, sha256 } from '../util/hash.mjs';
|
|
18
21
|
import { fingerprint } from '../../spec/lib/fingerprint.mjs';
|
|
22
|
+
import { recordedOriginSummary } from '../../spec/lib/origins.mjs';
|
|
19
23
|
import { persistenceSignalsFor } from '../supply/persistence.mjs';
|
|
24
|
+
import { collectOwnedManifests, hashNativeTestUniverse } from './universe.mjs';
|
|
20
25
|
|
|
21
26
|
|
|
22
27
|
/** Is `child` the same file as `root`, or under it? Both must already be real paths. */
|
|
@@ -51,6 +56,12 @@ export function selectClaims(claims, only) {
|
|
|
51
56
|
return only.map((id) => byId.get(id));
|
|
52
57
|
}
|
|
53
58
|
|
|
59
|
+
/** Reuse cannot stand in for a measurement under a different worker policy. */
|
|
60
|
+
export function sameWorkerPolicy(previousRun, { workers = 1, serial = workers === 1, runnerCommand } = {}) {
|
|
61
|
+
if (runnerCommand) return previousRun?.workers === undefined && !previousRun?.serial;
|
|
62
|
+
return previousRun?.workers === (serial ? 1 : workers) && Boolean(previousRun?.serial) === serial;
|
|
63
|
+
}
|
|
64
|
+
|
|
54
65
|
/**
|
|
55
66
|
* Probe every fault of every claim. Returns a spec-conformant evidence
|
|
56
67
|
* document; writing it is the caller's job.
|
|
@@ -63,7 +74,8 @@ export async function probe({
|
|
|
63
74
|
ref = 'HEAD',
|
|
64
75
|
refExplicit = false,
|
|
65
76
|
ignoreDirty = false,
|
|
66
|
-
|
|
77
|
+
workers = 1,
|
|
78
|
+
serial = workers === 1,
|
|
67
79
|
onWarn = () => {},
|
|
68
80
|
budgetMs = 120_000,
|
|
69
81
|
commandBudget,
|
|
@@ -80,6 +92,18 @@ export async function probe({
|
|
|
80
92
|
previous,
|
|
81
93
|
onProgress = () => {},
|
|
82
94
|
}) {
|
|
95
|
+
if (!Number.isSafeInteger(workers) || workers < 1) throw new PreconditionError('workers must be a positive safe integer');
|
|
96
|
+
resetInterpreterCache(); // Cache only within this measurement, never across commands.
|
|
97
|
+
const measuredStart = performance.now();
|
|
98
|
+
const startedAt = new Date().toISOString();
|
|
99
|
+
const measurements = { elapsedMs: 0, runnerMs: 0, overheadMs: 0, runnerInvocations: 0 };
|
|
100
|
+
const timedRun = async (runner, opts) => {
|
|
101
|
+
assertNotCancelled();
|
|
102
|
+
const start = performance.now();
|
|
103
|
+
measurements.runnerInvocations++;
|
|
104
|
+
try { return await runner.run(opts); }
|
|
105
|
+
finally { measurements.runnerMs += performance.now() - start; }
|
|
106
|
+
};
|
|
83
107
|
commandBudget?.assertOpen();
|
|
84
108
|
// realpath: git reports the repository root by its real path (/private/var
|
|
85
109
|
// on macOS, not /var); every relative() below must start from the same place.
|
|
@@ -183,7 +207,6 @@ export async function probe({
|
|
|
183
207
|
const contention = detectContention();
|
|
184
208
|
if (contention.detected) onWarn(contentionWarning(contention));
|
|
185
209
|
|
|
186
|
-
const startedAt = new Date().toISOString();
|
|
187
210
|
const iso = mode === 'worktree' ? createScratch({ repoRoot: root, projectDir, ref: snapshot ?? ref, scratchBase, nodeModules }) : inPlace({ repoRoot: root, projectDir });
|
|
188
211
|
const isoReal = realpathSync(iso.projectDir);
|
|
189
212
|
const records = [];
|
|
@@ -218,20 +241,16 @@ export async function probe({
|
|
|
218
241
|
// project runner is; it must resolve before the first such defender runs.
|
|
219
242
|
const owned = OWNED_RUNNERS.filter((r) => r !== runner
|
|
220
243
|
&& !(r.name === 'python' && ['python', 'pytest', 'unittest'].includes(runner.name)));
|
|
221
|
-
|
|
222
|
-
|
|
244
|
+
let ownedChecked = new Map();
|
|
245
|
+
let manifests = new Map(primaryManifest ? [[runner, primaryManifest]] : []);
|
|
223
246
|
if (!commandTemplate) {
|
|
224
|
-
|
|
225
|
-
|
|
226
|
-
|
|
227
|
-
|
|
228
|
-
|
|
229
|
-
|
|
230
|
-
|
|
231
|
-
} catch (error) {
|
|
232
|
-
throw new PreconditionError(`${r.name}: test discovery failed: ${error.message}`);
|
|
233
|
-
}
|
|
234
|
-
}
|
|
247
|
+
try {
|
|
248
|
+
({ ownedChecked, manifests } = await collectOwnedManifests(runner, primaryManifest, {
|
|
249
|
+
projectDir: iso.projectDir, sourceDir: projectDir, python, budgetMs,
|
|
250
|
+
budgetFor: () => commandBudget?.runBudget(budgetMs) ?? budgetMs,
|
|
251
|
+
assertOpen: () => commandBudget?.assertOpen(),
|
|
252
|
+
}));
|
|
253
|
+
} catch (error) { throw new PreconditionError(`test discovery failed: ${error.message}`); }
|
|
235
254
|
}
|
|
236
255
|
const ensureOwned = async (r, file) => {
|
|
237
256
|
if (!ownedChecked.has(r)) {
|
|
@@ -265,9 +284,7 @@ export async function probe({
|
|
|
265
284
|
: [...ownerByFile.keys()]).sort());
|
|
266
285
|
const testUniverseHash = commandTemplate
|
|
267
286
|
? sha256(JSON.stringify({ schemaVersion: 1, source: 'custom-command-static', files: allTests }))
|
|
268
|
-
:
|
|
269
|
-
.map(([r, manifest]) => ({ runner: r.name, hash: manifest.testUniverseHash }))
|
|
270
|
-
.sort((a, b) => a.runner.localeCompare(b.runner))));
|
|
287
|
+
: hashNativeTestUniverse(manifests);
|
|
271
288
|
const baselineCache = new Map();
|
|
272
289
|
const discoveryHashCache = new Map();
|
|
273
290
|
// The negative control is charged per (target file, defender set): two
|
|
@@ -277,6 +294,7 @@ export async function probe({
|
|
|
277
294
|
// runner — a `.py` source file is Python's whatever the project runs.
|
|
278
295
|
const fatalEditFor = (file) => runnerFor(iso.projectDir, file, runner).fatalEdit?.();
|
|
279
296
|
const prior = previous && previous.run.confirmRuns === confirmRuns
|
|
297
|
+
&& sameWorkerPolicy(previous.run, { workers, serial, runnerCommand })
|
|
280
298
|
? new Map(previous.records.map((r) => [`${r.claim.id}/${r.subject.id}`, r]))
|
|
281
299
|
: new Map();
|
|
282
300
|
const partitionConfigured = (files) => {
|
|
@@ -295,7 +313,7 @@ export async function probe({
|
|
|
295
313
|
// file the interpreter actually loaded (Python) reports it back.
|
|
296
314
|
const runDefenders = async (files, targets = []) => {
|
|
297
315
|
if (commandTemplate) {
|
|
298
|
-
const result = await runner
|
|
316
|
+
const result = await timedRun(runner, { projectDir: iso.projectDir, sourceDir: projectDir, python, files, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, commandTemplate, serial, workers, targets });
|
|
299
317
|
commandBudget?.assertOpen();
|
|
300
318
|
return result;
|
|
301
319
|
}
|
|
@@ -304,7 +322,7 @@ export async function probe({
|
|
|
304
322
|
for (const [r, group] of groups) {
|
|
305
323
|
commandBudget?.assertOpen();
|
|
306
324
|
if (r !== runner) await ensureOwned(r, group[0]);
|
|
307
|
-
parts.push(await r
|
|
325
|
+
parts.push(await timedRun(r, { projectDir: iso.projectDir, sourceDir: projectDir, python, files: group, budgetMs: commandBudget?.runBudget(budgetMs) ?? budgetMs, serial, workers, targets }));
|
|
308
326
|
commandBudget?.assertOpen();
|
|
309
327
|
}
|
|
310
328
|
return mergeRuns(parts);
|
|
@@ -373,6 +391,10 @@ export async function probe({
|
|
|
373
391
|
|
|
374
392
|
commandBudget?.assertOpen();
|
|
375
393
|
|
|
394
|
+
assertNotCancelled();
|
|
395
|
+
const dirty = isDirty(root);
|
|
396
|
+
measurements.elapsedMs = performance.now() - measuredStart;
|
|
397
|
+
measurements.overheadMs = measurements.elapsedMs - measurements.runnerMs;
|
|
376
398
|
return {
|
|
377
399
|
schemaVersion: 1,
|
|
378
400
|
tool: { name: 'testguard', version: toolVersion },
|
|
@@ -380,11 +402,13 @@ export async function probe({
|
|
|
380
402
|
id: `run-${startedAt.replace(/[-:.]/g, '').slice(0, 15)}`,
|
|
381
403
|
startedAt,
|
|
382
404
|
finishedAt: new Date().toISOString(),
|
|
383
|
-
repo: { head, dirty
|
|
405
|
+
repo: { head, dirty, ...(snapshot ? { snapshot } : {}), ...(ignoredDirty.length ? { ignoredDirty } : {}) },
|
|
384
406
|
runner: { name: labelOf(runner), ...((runnerVersion ?? readRunnerVersion(projectDir, runner.name)) ? { version: runnerVersion ?? readRunnerVersion(projectDir, runner.name) } : {}) },
|
|
385
407
|
...(runnersUsed.size > 1 ? { runners: [...runnersUsed].map(([n, v]) => ({ name: n, ...(v ? { version: v } : {}) })) } : {}),
|
|
386
408
|
confirmRuns,
|
|
387
|
-
|
|
409
|
+
measurements,
|
|
410
|
+
...(!runnerCommand ? { workers: serial ? 1 : workers } : {}),
|
|
411
|
+
...(serial && !runnerCommand ? { serial: true } : {}),
|
|
388
412
|
...(contention.detected ? { contention } : {}),
|
|
389
413
|
...(confirmRuns < 3 ? { provisional: true } : {}),
|
|
390
414
|
// The isolation that was actually created, not the mode that was asked
|
|
@@ -394,6 +418,7 @@ export async function probe({
|
|
|
394
418
|
mode: iso.mode,
|
|
395
419
|
},
|
|
396
420
|
records,
|
|
421
|
+
origins: recordedOriginSummary(records),
|
|
397
422
|
};
|
|
398
423
|
}
|
|
399
424
|
|
|
@@ -520,7 +545,7 @@ async function probeOne({ claim, fault, defenders, discovered, allTests, iso, is
|
|
|
520
545
|
const subject = subjectOf(fault, sha256);
|
|
521
546
|
|
|
522
547
|
// Same source, same defenders, same N, same fault: the verdict cannot have changed.
|
|
523
|
-
if (isReusable(prior, { inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
548
|
+
if (isReusable(prior, { claim, inputs, ...defenderSelection(claim, fault), resolved: defenders, contentHash: subject.contentHash })) {
|
|
524
549
|
return { ...prior, defenders: { ...prior.defenders, ...defenderSelection(claim, fault) }, reusedFrom: prior.reusedFrom ?? priorRunId };
|
|
525
550
|
}
|
|
526
551
|
|
|
@@ -1,3 +1,4 @@
|
|
|
1
|
+
import { registerChild } from './lifecycle.mjs';
|
|
1
2
|
import { spawn } from 'node:child_process';
|
|
2
3
|
import { randomBytes } from 'node:crypto';
|
|
3
4
|
import { closeSync, constants, fstatSync, lstatSync, openSync, readSync, realpathSync, rmSync, statSync } from 'node:fs';
|
|
@@ -177,6 +178,7 @@ export function runDiscoveryProcess({
|
|
|
177
178
|
return;
|
|
178
179
|
}
|
|
179
180
|
|
|
181
|
+
registerChild(child, () => terminateProcessTree(child));
|
|
180
182
|
const started = Date.now();
|
|
181
183
|
const stdout = [];
|
|
182
184
|
const stderr = [];
|
|
@@ -38,8 +38,8 @@ export async function discoverTests({ projectDir, version, timeoutMs, maxOutputB
|
|
|
38
38
|
}
|
|
39
39
|
// --runTestsByPath: positionals are exact paths, not regexes — a path with `+` or `(` would otherwise silently match nothing.
|
|
40
40
|
/** The command line, as data. Exported so it is falsifiable without spawning anything. */
|
|
41
|
-
export const argvFor = (projectDir, files, outFile, { serial = false } = {}) =>
|
|
42
|
-
[...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial ? ['--runInBand'] : []), '--runTestsByPath', ...files];
|
|
41
|
+
export const argvFor = (projectDir, files, outFile, { serial = false, workers = 1 } = {}) =>
|
|
42
|
+
[...runnerArgv(projectDir, 'jest', 'jest'), '--ci', '--json', `--outputFile=${outFile}`, ...(serial || workers === 1 ? ['--runInBand'] : [`--maxWorkers=${workers}`]), '--runTestsByPath', ...files];
|
|
43
43
|
export const run = (opts) => runProcess({ ...opts, argv: (files, outFile) => argvFor(opts.projectDir, files, outFile, opts) });
|
|
44
44
|
|
|
45
45
|
/**
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
/** Own only children and cleanups registered by this TestGuard process. */
|
|
2
|
+
const children = new Map();
|
|
3
|
+
const cleanups = new Map();
|
|
4
|
+
let installed = false;
|
|
5
|
+
let cancelling = false;
|
|
6
|
+
|
|
7
|
+
function cleanup() {
|
|
8
|
+
// Restore mutations before removing isolation, even across successive probes.
|
|
9
|
+
for (const [fn] of [...cleanups].reverse().sort((a, b) => b[1] - a[1])) {
|
|
10
|
+
try { fn(); } catch (error) { process.stderr.write(`cleanup failed: ${error.message}\n`); }
|
|
11
|
+
}
|
|
12
|
+
cleanups.clear();
|
|
13
|
+
}
|
|
14
|
+
|
|
15
|
+
function install() {
|
|
16
|
+
if (installed) return;
|
|
17
|
+
installed = true;
|
|
18
|
+
for (const signal of ['SIGINT', 'SIGTERM', 'SIGHUP']) {
|
|
19
|
+
process.on(signal, async () => {
|
|
20
|
+
if (cancelling) return;
|
|
21
|
+
cancelling = true;
|
|
22
|
+
const waits = [...children].map(([child, stop]) => {
|
|
23
|
+
const closed = new Promise((resolve) => child.once('close', resolve));
|
|
24
|
+
stop();
|
|
25
|
+
return closed;
|
|
26
|
+
});
|
|
27
|
+
let timer;
|
|
28
|
+
await Promise.race([
|
|
29
|
+
Promise.allSettled(waits),
|
|
30
|
+
new Promise((resolve) => { timer = setTimeout(resolve, 1000); }),
|
|
31
|
+
]);
|
|
32
|
+
clearTimeout(timer);
|
|
33
|
+
cleanup();
|
|
34
|
+
process.exit(130);
|
|
35
|
+
});
|
|
36
|
+
}
|
|
37
|
+
process.on('exit', () => {
|
|
38
|
+
for (const stop of children.values()) stop();
|
|
39
|
+
cleanup();
|
|
40
|
+
});
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export function registerChild(child, stop) {
|
|
44
|
+
install();
|
|
45
|
+
children.set(child, stop);
|
|
46
|
+
child.once('close', () => children.delete(child));
|
|
47
|
+
if (cancelling) stop();
|
|
48
|
+
}
|
|
49
|
+
|
|
50
|
+
export function registerCleanup(fn, { priority = 0 } = {}) {
|
|
51
|
+
install();
|
|
52
|
+
cleanups.set(fn, priority);
|
|
53
|
+
return () => cleanups.delete(fn);
|
|
54
|
+
}
|
|
55
|
+
|
|
56
|
+
export function assertNotCancelled() {
|
|
57
|
+
if (cancelling) throw new Error('TestGuard command cancelled; no partial result may be published');
|
|
58
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
import { existsSync, readFileSync } from 'node:fs';
|
|
2
2
|
import { createRequire } from 'node:module';
|
|
3
3
|
import { isAbsolute, join, relative, resolve, sep } from 'node:path';
|
|
4
|
-
import {
|
|
4
|
+
import { runProcess, listTestFiles, firstInformativeLine, runnerArgv } from './shared.mjs';
|
|
5
5
|
import { createDiscoveryManifest, DiscoveryError, hashDiscoveryConfigs, normalizeDiscoveredFiles, runDiscoveryReportProcess } from './discovery.mjs';
|
|
6
6
|
import { globToRegExp } from '../../util/glob.mjs';
|
|
7
7
|
|
|
@@ -185,7 +185,7 @@ export function parseReport(report, durationMs) {
|
|
|
185
185
|
|
|
186
186
|
export const run = (opts) => {
|
|
187
187
|
const env = { PLAYWRIGHT_JSON_OUTPUT_FILE: '{out}', PLAYWRIGHT_JSON_OUTPUT_NAME: '{out}' };
|
|
188
|
-
return runProcess({ ...opts, env, parse: parseReport, argv: (files) =>
|
|
188
|
+
return runProcess({ ...opts, env, parse: parseReport, argv: (files) => argvFor(opts.projectDir, files, opts) });
|
|
189
189
|
};
|
|
190
190
|
|
|
191
191
|
/**
|
|
@@ -196,3 +196,7 @@ export const run = (opts) => {
|
|
|
196
196
|
* contained before.
|
|
197
197
|
*/
|
|
198
198
|
export const fatalEdit = () => '/* testguard negative control: this file must not parse */\n(\n';
|
|
199
|
+
|
|
200
|
+
/** Run the project-pinned CLI directly, avoiding an npm subprocess per confirmation. */
|
|
201
|
+
export const argvFor = (projectDir, files, { serial = false, workers = 1 } = {}) =>
|
|
202
|
+
[...runnerArgv(projectDir, '@playwright/test', 'playwright'), 'test', '--reporter=json', `--workers=${serial ? 1 : workers}`, ...files];
|
|
@@ -1,4 +1,6 @@
|
|
|
1
|
+
import { performance } from 'node:perf_hooks';
|
|
1
2
|
import { existsSync, readFileSync } from 'node:fs';
|
|
3
|
+
import { registerChild } from './lifecycle.mjs';
|
|
2
4
|
import { spawn } from 'node:child_process';
|
|
3
5
|
import { dirname, join } from 'node:path';
|
|
4
6
|
import { fileURLToPath } from 'node:url';
|
|
@@ -120,9 +122,10 @@ function ask(path, budgetMs) {
|
|
|
120
122
|
} catch {
|
|
121
123
|
return resolve(null);
|
|
122
124
|
}
|
|
125
|
+
registerChild(child, () => terminateProcessTree(child));
|
|
123
126
|
let out = '';
|
|
124
127
|
child.on('error', () => resolve(null));
|
|
125
|
-
child.stdout.on('data', (d) => (out
|
|
128
|
+
child.stdout.on('data', (d) => (out = (out + d).slice(-65536)));
|
|
126
129
|
child.stderr.on('data', () => {});
|
|
127
130
|
const timer = setTimeout(() => terminateProcessTree(child), budgetMs);
|
|
128
131
|
child.on('close', (code) => {
|
|
@@ -276,10 +279,10 @@ export function environment(targets) {
|
|
|
276
279
|
* the configured value; the rest of the project's `addopts` is left alone,
|
|
277
280
|
* because TestGuard runs a project's tests the way the project runs them.
|
|
278
281
|
*/
|
|
279
|
-
export function argvFor({ engine, interpreter, files, serial }) {
|
|
282
|
+
export function argvFor({ engine, interpreter, files, serial = true }) {
|
|
280
283
|
if (engine === 'pytest') {
|
|
281
284
|
return [interpreter, '-m', 'pytest', '-p', '_testguard_pytest_plugin', '-p', 'no:cacheprovider',
|
|
282
|
-
'--maxfail=0', '-q',
|
|
285
|
+
'--maxfail=0', '-q', '-p', 'no:xdist', ...files];
|
|
283
286
|
}
|
|
284
287
|
return [interpreter, '-m', '_testguard_unittest_main', ...files];
|
|
285
288
|
}
|
|
@@ -287,12 +290,15 @@ export function argvFor({ engine, interpreter, files, serial }) {
|
|
|
287
290
|
export function makeRun(pinned) {
|
|
288
291
|
const check = makeCheck(pinned);
|
|
289
292
|
return async (opts) => {
|
|
290
|
-
const
|
|
293
|
+
const deadline = performance.now() + (opts.budgetMs ?? 120_000);
|
|
294
|
+
const remaining = () => Math.max(1, Math.floor(deadline - performance.now()));
|
|
295
|
+
const resolved = await check({ projectDir: opts.sourceDir ?? opts.projectDir, python: opts.python, budgetMs: remaining(), budgetFor: remaining });
|
|
291
296
|
if (!resolved.ok) {
|
|
292
297
|
return { run: { outcome: 'error', tests: { total: 0, passed: 0, failed: 0 }, assertionFailures: 0, durationMs: 0 }, timeouts: 0, loadMessage: resolved.message, failedTests: [], provenance: {} };
|
|
293
298
|
}
|
|
294
299
|
return runProcess({
|
|
295
300
|
...opts,
|
|
301
|
+
budgetMs: remaining(),
|
|
296
302
|
env: environment(opts.targets),
|
|
297
303
|
parse: parseReport,
|
|
298
304
|
argv: (files) => argvFor({ engine: resolved.engine, interpreter: resolved.interpreter, files, serial: opts.serial }),
|