rcf-lite 0.19.0 → 0.20.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/CHANGELOG.md +15 -0
- package/bin/rcf.js +5 -0
- package/fixtures/canary-manifest.json +6 -6
- package/package.json +2 -2
- package/rcf/code-nodes/cn-098.json +13 -0
- package/rcf/code-nodes/cn-099.json +13 -0
- package/rcf/code-nodes/cn-100.json +13 -0
- package/rcf/code-nodes/cn-101.json +13 -0
- package/rcf/code-nodes/cn-102.json +13 -0
- package/rcf/code-nodes/cn-103.json +13 -0
- package/rcf/code-nodes/cn-104.json +13 -0
- package/rcf/code-nodes/cn-105.json +13 -0
- package/rcf/evals/eval-001.json +55 -0
- package/rcf/fbs/fbs-035.json +18 -0
- package/rcf/requirements/req-016.json +37 -0
- package/rcf/test-suites/ts-045.json +46 -0
- package/rcf/user-stories/us-1601.json +37 -0
- package/releases/releases.yaml +10 -1
- package/src/cli/create.js +14 -0
- package/src/cli/eval-coverage.js +221 -0
- package/src/cli/eval.js +43 -0
- package/src/cli/finalise.js +64 -0
- package/src/cli/help.js +4 -0
- package/src/core/store/ids.js +5 -1
- package/src/core/store/init.js +4 -0
- package/src/core/store/loader.js +4 -0
- package/src/core/store/validator.js +8 -1
- package/src/core/store/walker.js +71 -2
- package/src/core/store/writer.js +4 -0
- package/src/eval/judge.js +338 -0
- package/src/finalise/index.js +8 -0
- package/src/finalise/ingest.js +27 -0
- package/src/finalise/ship-without-eval.js +123 -0
- package/src/query/eval-coverage.js +162 -0
- package/src/verify/chain/index.js +67 -0
- package/src/verify/cli/run.js +15 -0
- package/src/verify/engine/index.js +10 -0
- package/src/verify/verdict/index.js +46 -0
|
@@ -0,0 +1,221 @@
|
|
|
1
|
+
// `rcf audit eval coverage` subcommand handler. Sibling of `rcf audit
|
|
2
|
+
// coverage`; grades non-deterministic AC coverage by EVAL docs rather
|
|
3
|
+
// than TS/TC coverage. See spec section 4 of
|
|
4
|
+
// projects/rcf-lite-wsd/specs/rcf-eval-node-spec-2026-09-04.md.
|
|
5
|
+
//
|
|
6
|
+
// Coverage rule (spec 2.3): an AC counts as covered by EVAL only when
|
|
7
|
+
// at least one EVAL document lists its id in acIds[], is not
|
|
8
|
+
// superseded, and has a non-pending runRecord[] entry (a "resolving"
|
|
9
|
+
// EVAL, mirroring the resolving-TC rule on `rcf audit coverage`).
|
|
10
|
+
//
|
|
11
|
+
// Exit codes: 0 clean, 2 usage refusal, 3 tree drift, 4 strict-gate
|
|
12
|
+
// refusal (any nonDeterministic AC without a resolving EVAL).
|
|
13
|
+
//
|
|
14
|
+
// Deterministic ACs never trigger the strict gate. A subtree with zero
|
|
15
|
+
// nonDeterministic ACs passes trivially. Presence of a resolving EVAL
|
|
16
|
+
// on a deterministic AC is reported as `covered-optional`, never as a
|
|
17
|
+
// defect.
|
|
18
|
+
|
|
19
|
+
import { parseArgs } from 'node:util';
|
|
20
|
+
|
|
21
|
+
import { formatErrors } from '#core/errors';
|
|
22
|
+
import { walkTree } from '#core/store';
|
|
23
|
+
import { findProjectRoot } from '../view/index.js';
|
|
24
|
+
import {
|
|
25
|
+
classifyCoverageScope,
|
|
26
|
+
} from '../query/index.js';
|
|
27
|
+
import { computeEvalCoverage } from '../query/eval-coverage.js';
|
|
28
|
+
|
|
29
|
+
const OPTION_SPEC = {
|
|
30
|
+
strict: { type: 'boolean' },
|
|
31
|
+
format: { type: 'string' },
|
|
32
|
+
help: { type: 'boolean' },
|
|
33
|
+
// 0.7.0 verification-integrity precedent: opt-in extra gate on
|
|
34
|
+
// --strict that refuses any EVAL still at authoringStatus `draft`.
|
|
35
|
+
// Named --require-approved to match `rcf audit coverage`'s flag,
|
|
36
|
+
// spec section 12 Q1 recommendation.
|
|
37
|
+
'require-approved': { type: 'boolean' },
|
|
38
|
+
};
|
|
39
|
+
|
|
40
|
+
export const HELP = `Usage: rcf audit eval coverage [scope-id] [options]
|
|
41
|
+
|
|
42
|
+
Report EVAL coverage of the non-deterministic ACs on the scoped
|
|
43
|
+
subtree. An AC counts as covered by EVAL only when at least one EVAL
|
|
44
|
+
document lists its id in acIds[], is not superseded, and has a
|
|
45
|
+
non-pending runRecord[] entry (a "resolving" EVAL, mirroring the
|
|
46
|
+
resolving-TC rule on \`rcf audit coverage\`).
|
|
47
|
+
|
|
48
|
+
Deterministic ACs never trigger the strict gate. Presence of a
|
|
49
|
+
resolving EVAL on a deterministic AC is reported as covered-optional,
|
|
50
|
+
never as a defect.
|
|
51
|
+
|
|
52
|
+
Positional:
|
|
53
|
+
scope-id Optional PRD / REQ / US id to scope the
|
|
54
|
+
audit to a subtree. Below-AC ids exit 2.
|
|
55
|
+
|
|
56
|
+
Options:
|
|
57
|
+
--strict Per-AC-strict mode; exits 4 on any
|
|
58
|
+
nonDeterministic AC without a resolving
|
|
59
|
+
EVAL. Deterministic ACs are never gated.
|
|
60
|
+
--require-approved Extra --strict gate: refuse any EVAL still
|
|
61
|
+
at authoringStatus 'draft'.
|
|
62
|
+
--format <format> table (default) | json | mermaid
|
|
63
|
+
--help Print this help
|
|
64
|
+
`;
|
|
65
|
+
|
|
66
|
+
const VALID_FORMATS = new Set(['table', 'json', 'mermaid']);
|
|
67
|
+
|
|
68
|
+
/**
|
|
69
|
+
* @param {string[]} argv - argv slice after `eval coverage`
|
|
70
|
+
* @param {object} [deps]
|
|
71
|
+
* @returns {Promise<number>}
|
|
72
|
+
*/
|
|
73
|
+
export async function main(argv, deps = {}) {
|
|
74
|
+
const stdout = deps.stdout ?? process.stdout;
|
|
75
|
+
const stderr = deps.stderr ?? process.stderr;
|
|
76
|
+
const cwd = deps.cwd ?? process.cwd();
|
|
77
|
+
|
|
78
|
+
let parsed;
|
|
79
|
+
try {
|
|
80
|
+
parsed = parseArgs({ args: argv, options: OPTION_SPEC, allowPositionals: true, strict: true });
|
|
81
|
+
} catch (err) {
|
|
82
|
+
stderr.write(`[error] usage ${err.message}\n`);
|
|
83
|
+
stderr.write(HELP);
|
|
84
|
+
return 2;
|
|
85
|
+
}
|
|
86
|
+
const flags = parsed.values;
|
|
87
|
+
const positionals = parsed.positionals;
|
|
88
|
+
if (flags.help) { stdout.write(HELP); return 0; }
|
|
89
|
+
|
|
90
|
+
const format = flags.format ?? 'table';
|
|
91
|
+
if (!VALID_FORMATS.has(format)) {
|
|
92
|
+
stderr.write(`[error] usage eval coverage: unknown --format ${format} (expected table | json | mermaid)\n`);
|
|
93
|
+
return 2;
|
|
94
|
+
}
|
|
95
|
+
if (positionals.length > 1) {
|
|
96
|
+
stderr.write('[error] usage eval coverage: multiple positional ids are not supported\n');
|
|
97
|
+
return 2;
|
|
98
|
+
}
|
|
99
|
+
|
|
100
|
+
const projectRoot = await findProjectRoot(cwd);
|
|
101
|
+
if (!projectRoot) {
|
|
102
|
+
stderr.write('[error] usage no project root found (no rcf/manifest.json in this directory or any ancestor). Run `npx rcf init` to create and wire a project.\n');
|
|
103
|
+
return 2;
|
|
104
|
+
}
|
|
105
|
+
const { tree, errors } = await walkTree({ projectRoot });
|
|
106
|
+
if (errors.length > 0) {
|
|
107
|
+
stderr.write(`${formatErrors(errors, { verbose: false, strict: false })}\n`);
|
|
108
|
+
return 3;
|
|
109
|
+
}
|
|
110
|
+
|
|
111
|
+
let scopeId = null;
|
|
112
|
+
if (positionals.length === 1) {
|
|
113
|
+
scopeId = positionals[0];
|
|
114
|
+
if (scopeId.includes('*') || scopeId.includes('?')) {
|
|
115
|
+
stderr.write('[error] usage eval coverage: wildcard / glob positional not supported\n');
|
|
116
|
+
return 2;
|
|
117
|
+
}
|
|
118
|
+
const classification = classifyCoverageScope(tree, scopeId);
|
|
119
|
+
if (classification === 'below-ac') {
|
|
120
|
+
stderr.write(
|
|
121
|
+
`[error] usage eval coverage: scope-id ${scopeId} is below the AC layer or off the REQ chain; ` +
|
|
122
|
+
'eval coverage scope must be a PRD / REQ / US id\n',
|
|
123
|
+
);
|
|
124
|
+
return 2;
|
|
125
|
+
}
|
|
126
|
+
if (classification === 'not-found' || classification === 'unknown-kind') {
|
|
127
|
+
stderr.write(`[error] usage eval coverage: id ${scopeId} not found\n`);
|
|
128
|
+
return 2;
|
|
129
|
+
}
|
|
130
|
+
}
|
|
131
|
+
|
|
132
|
+
const report = computeEvalCoverage(tree, { scopeId });
|
|
133
|
+
|
|
134
|
+
if (format === 'json') stdout.write(`${JSON.stringify(report, null, 2)}\n`);
|
|
135
|
+
else if (format === 'mermaid') stdout.write(formatMermaid(report));
|
|
136
|
+
else stdout.write(formatTable(report));
|
|
137
|
+
|
|
138
|
+
// --strict on any missing = exit 4 (CI-gate friendly). Otherwise 0.
|
|
139
|
+
if (flags.strict && !report.ok) return 4;
|
|
140
|
+
|
|
141
|
+
if (flags.strict && flags['require-approved']) {
|
|
142
|
+
const draftEvalIds = (tree.evals ?? []).filter((e) => e.status !== 'approved').map((e) => e.id);
|
|
143
|
+
if (draftEvalIds.length > 0) {
|
|
144
|
+
stderr.write(`[error] eval coverage --strict --require-approved: refused - ${draftEvalIds.length} EVAL(s) still not approved: ${draftEvalIds.join(', ')}\n`);
|
|
145
|
+
return 4;
|
|
146
|
+
}
|
|
147
|
+
}
|
|
148
|
+
return 0;
|
|
149
|
+
}
|
|
150
|
+
|
|
151
|
+
/**
|
|
152
|
+
* Render the EVAL coverage report as a plain-text table.
|
|
153
|
+
*
|
|
154
|
+
* @param {import('../query/eval-coverage.js').EvalCoverageReport} report
|
|
155
|
+
* @returns {string}
|
|
156
|
+
*/
|
|
157
|
+
export function formatTable(report) {
|
|
158
|
+
const lines = [];
|
|
159
|
+
lines.push('EVAL coverage report');
|
|
160
|
+
if (report.scopeId) lines.push(`scope: ${report.scopeId}`);
|
|
161
|
+
lines.push('');
|
|
162
|
+
if (report.nonDeterministicCount === 0) {
|
|
163
|
+
lines.push('no nonDeterministic ACs on this chain; audit passes trivially');
|
|
164
|
+
lines.push('');
|
|
165
|
+
lines.push(`ok=${report.ok}`);
|
|
166
|
+
return `${lines.join('\n')}\n`;
|
|
167
|
+
}
|
|
168
|
+
lines.push(' AC US determinism evalStatus outcome evalId');
|
|
169
|
+
for (const ac of report.acs) {
|
|
170
|
+
const cols = [
|
|
171
|
+
pad(ac.acId, 15),
|
|
172
|
+
pad(ac.usId, 9),
|
|
173
|
+
pad(ac.determinism, 16),
|
|
174
|
+
pad(ac.evalStatus, 11),
|
|
175
|
+
pad(ac.outcome, 18),
|
|
176
|
+
ac.evalId ?? '',
|
|
177
|
+
];
|
|
178
|
+
lines.push(` ${cols.join(' ')}`);
|
|
179
|
+
}
|
|
180
|
+
lines.push('');
|
|
181
|
+
lines.push(
|
|
182
|
+
`nonDeterministic=${report.nonDeterministicCount}, `
|
|
183
|
+
+ `covered=${report.coveredCount}, `
|
|
184
|
+
+ `missing=${report.missingCount}, ok=${report.ok}`,
|
|
185
|
+
);
|
|
186
|
+
return `${lines.join('\n')}\n`;
|
|
187
|
+
}
|
|
188
|
+
|
|
189
|
+
/**
|
|
190
|
+
* Colour-coded mermaid summary. Spec section 4: green deterministic
|
|
191
|
+
* (out of scope), amber nonDeterministic with a resolving EVAL, red
|
|
192
|
+
* nonDeterministic without one.
|
|
193
|
+
*
|
|
194
|
+
* @param {import('../query/eval-coverage.js').EvalCoverageReport} report
|
|
195
|
+
* @returns {string}
|
|
196
|
+
*/
|
|
197
|
+
export function formatMermaid(report) {
|
|
198
|
+
const lines = ['graph TD'];
|
|
199
|
+
lines.push(' classDef deterministic fill:#d1f2c4,stroke:#2f7a1e;');
|
|
200
|
+
lines.push(' classDef covered fill:#ffe6a1,stroke:#a56a00;');
|
|
201
|
+
lines.push(' classDef missing fill:#f4b4b4,stroke:#a00000;');
|
|
202
|
+
for (const ac of report.acs) {
|
|
203
|
+
const cls = ac.outcome === 'missing'
|
|
204
|
+
? 'missing'
|
|
205
|
+
: ac.outcome === 'covered'
|
|
206
|
+
? 'covered'
|
|
207
|
+
: 'deterministic';
|
|
208
|
+
const label = ac.evalId ? `${ac.acId}\\n[${ac.evalId}]` : ac.acId;
|
|
209
|
+
lines.push(` ${idOf(ac.acId)}["${label}"]:::${cls}`);
|
|
210
|
+
}
|
|
211
|
+
return `${lines.join('\n')}\n`;
|
|
212
|
+
}
|
|
213
|
+
|
|
214
|
+
function pad(s, w) {
|
|
215
|
+
const str = String(s ?? '');
|
|
216
|
+
return str.length >= w ? str : str + ' '.repeat(w - str.length);
|
|
217
|
+
}
|
|
218
|
+
|
|
219
|
+
function idOf(s) {
|
|
220
|
+
return String(s).replace(/[^A-Za-z0-9]+/g, '_');
|
|
221
|
+
}
|
package/src/cli/eval.js
ADDED
|
@@ -0,0 +1,43 @@
|
|
|
1
|
+
// `rcf audit eval <sub-verb>` dispatcher. v1 exposes one sub-verb:
|
|
2
|
+
// `coverage`. The compound invocation `rcf audit eval coverage`
|
|
3
|
+
// matches the spec text exactly (section 4). Future sibling sub-verbs
|
|
4
|
+
// (`rcf audit eval trace`, `rcf build eval run`, ...) plug in here.
|
|
5
|
+
|
|
6
|
+
import { main as evalCoverageMain, HELP as COVERAGE_HELP } from './eval-coverage.js';
|
|
7
|
+
|
|
8
|
+
export const HELP = `Usage: rcf audit eval <sub-verb> [options]
|
|
9
|
+
|
|
10
|
+
Sub-verbs:
|
|
11
|
+
coverage Report EVAL coverage over the chain.
|
|
12
|
+
|
|
13
|
+
Options:
|
|
14
|
+
--help Print this help
|
|
15
|
+
`;
|
|
16
|
+
|
|
17
|
+
const SUB_VERBS = {
|
|
18
|
+
coverage: evalCoverageMain,
|
|
19
|
+
};
|
|
20
|
+
|
|
21
|
+
/**
|
|
22
|
+
* @param {string[]} argv - argv slice after `eval`
|
|
23
|
+
* @param {object} [deps]
|
|
24
|
+
* @returns {Promise<number>}
|
|
25
|
+
*/
|
|
26
|
+
export async function main(argv, deps = {}) {
|
|
27
|
+
const stdout = deps.stdout ?? process.stdout;
|
|
28
|
+
const stderr = deps.stderr ?? process.stderr;
|
|
29
|
+
if (argv.length === 0 || argv[0] === '--help' || argv[0] === '-h') {
|
|
30
|
+
stdout.write(HELP);
|
|
31
|
+
return 0;
|
|
32
|
+
}
|
|
33
|
+
const subVerb = argv[0];
|
|
34
|
+
const handler = SUB_VERBS[subVerb];
|
|
35
|
+
if (!handler) {
|
|
36
|
+
stderr.write(`[error] usage unknown sub-verb '${subVerb}' under 'audit eval'.\n`);
|
|
37
|
+
stderr.write(HELP);
|
|
38
|
+
return 2;
|
|
39
|
+
}
|
|
40
|
+
return await handler(argv.slice(1), deps);
|
|
41
|
+
}
|
|
42
|
+
|
|
43
|
+
export { COVERAGE_HELP };
|
package/src/cli/finalise.js
CHANGED
|
@@ -37,13 +37,17 @@ import { kindOf } from '../query/index.js';
|
|
|
37
37
|
import {
|
|
38
38
|
buildVerifyArgs,
|
|
39
39
|
composeShipWithoutVerifiedRecord,
|
|
40
|
+
composeShipWithoutEvalRecord,
|
|
40
41
|
detectVerify,
|
|
42
|
+
findEvalRefusalAcs,
|
|
41
43
|
findMockOnlyDeclaredAcs,
|
|
42
44
|
loadReport,
|
|
45
|
+
reportHasEvalRefusal,
|
|
43
46
|
reportHasMockOnlyDeclared,
|
|
44
47
|
resolveAbsentVerify,
|
|
45
48
|
spawnVerify,
|
|
46
49
|
summariseReport,
|
|
50
|
+
writeShipWithoutEvalRecord,
|
|
47
51
|
writeShipWithoutVerifiedRecord,
|
|
48
52
|
} from '../finalise/index.js';
|
|
49
53
|
|
|
@@ -74,6 +78,11 @@ const OPTION_SPEC = {
|
|
|
74
78
|
// ship the FBS 'complete' without promoting to 'verified'. Records
|
|
75
79
|
// the operator's ack on the manifest.
|
|
76
80
|
'ship-without-verified': { type: 'boolean' },
|
|
81
|
+
// rcf-eval-node spec 2026-09-04 sections 5.2 + 8: sister opt-out for
|
|
82
|
+
// EVAL-MISSING / EVAL-BELOW-THRESHOLD. Takes a mandatory reason
|
|
83
|
+
// string (spec section 8: missing reason exits 2). Shape mirrors
|
|
84
|
+
// `--provision`: a value that looks like a flag is refused.
|
|
85
|
+
'ship-without-eval': { type: 'string' },
|
|
77
86
|
quiet: { type: 'boolean' },
|
|
78
87
|
help: { type: 'boolean' },
|
|
79
88
|
};
|
|
@@ -123,6 +132,13 @@ Options:
|
|
|
123
132
|
'complete'. Without this flag, the gate
|
|
124
133
|
refuses to promote when any AC lands in
|
|
125
134
|
those verdicts.
|
|
135
|
+
--ship-without-eval "..." Acknowledge EVAL-MISSING /
|
|
136
|
+
EVAL-BELOW-THRESHOLD per-AC verdicts and
|
|
137
|
+
proceed with an audit-log entry
|
|
138
|
+
(rcf-eval-node spec section 5.2). Reason
|
|
139
|
+
string is mandatory; a missing reason exits
|
|
140
|
+
2. The ack lands on the manifest under
|
|
141
|
+
shipWithoutEval[] with a monotonic id.
|
|
126
142
|
--quiet Suppress non-error confirmations
|
|
127
143
|
--help Print this help
|
|
128
144
|
|
|
@@ -345,6 +361,54 @@ export async function main(argv, deps = {}) {
|
|
|
345
361
|
stderr.write(`Report: ${outPath}\n`);
|
|
346
362
|
return 4;
|
|
347
363
|
}
|
|
364
|
+
// rcf-eval-node spec section 5.2: EVAL-MISSING / EVAL-BELOW-THRESHOLD
|
|
365
|
+
// refusal, sister of the mock-only refusal above. Opt-out via
|
|
366
|
+
// --ship-without-eval "<reason>"; missing reason exits 2, matching
|
|
367
|
+
// the ship-without-verified argv-shape lesson.
|
|
368
|
+
if (passLoaded.ok && reportHasEvalRefusal(passLoaded.report)) {
|
|
369
|
+
const evalDeclared = findEvalRefusalAcs(passLoaded.report);
|
|
370
|
+
const missingAcIds = evalDeclared.filter((e) => e.verdict === 'EVAL-MISSING').map((e) => e.acId);
|
|
371
|
+
const belowAcIds = evalDeclared.filter((e) => e.verdict === 'EVAL-BELOW-THRESHOLD').map((e) => e.acId);
|
|
372
|
+
const swe = flags['ship-without-eval'];
|
|
373
|
+
if (swe !== undefined) {
|
|
374
|
+
// A --ship-without-eval that looks like a flag is a swallowed
|
|
375
|
+
// next option; refuse it (parallel to the --provision guard).
|
|
376
|
+
if (typeof swe !== 'string' || swe.startsWith('-') || swe.trim().length === 0) {
|
|
377
|
+
stderr.write('[error] usage finalise: --ship-without-eval requires a reason string\n');
|
|
378
|
+
return 2;
|
|
379
|
+
}
|
|
380
|
+
const evalAck = composeShipWithoutEvalRecord({
|
|
381
|
+
manifest: tree.manifest,
|
|
382
|
+
fbsId,
|
|
383
|
+
reason: swe,
|
|
384
|
+
declaredAcs: evalDeclared,
|
|
385
|
+
reportPath: outPath,
|
|
386
|
+
});
|
|
387
|
+
const evalAckResult = await writeShipWithoutEvalRecord({
|
|
388
|
+
projectRoot, tree, record: evalAck,
|
|
389
|
+
});
|
|
390
|
+
if (isRcfError(evalAckResult)) {
|
|
391
|
+
if (evalAckResult.kind === 'ioFailure') { writeUnexpectedFailure(evalAckResult, stderr); return 1; }
|
|
392
|
+
stderr.write(`[error] ${evalAckResult.kind} ${evalAckResult.message}\n`);
|
|
393
|
+
if (evalAckResult.kind === 'validation' || evalAckResult.kind === 'brokenReference') return 3;
|
|
394
|
+
return 1;
|
|
395
|
+
}
|
|
396
|
+
if (!quiet) {
|
|
397
|
+
stdout.write(`[finalise] gate passed and ${evalDeclared.length} AC(s) came back with EVAL refusals; --ship-without-eval acknowledged (${evalAck.id} on rcf/manifest.json, reason: ${swe}); ${fbsId} left '${currentStatus}'. Report: ${outPath}\n`);
|
|
398
|
+
stdout.write(summariseReport(passLoaded.report));
|
|
399
|
+
}
|
|
400
|
+
return 0;
|
|
401
|
+
}
|
|
402
|
+
if (missingAcIds.length > 0) {
|
|
403
|
+
stderr.write(`finalise refused: EVAL missing on AC(s) ${missingAcIds.join(', ')}; author an EVAL or --ship-without-eval "reason"\n`);
|
|
404
|
+
}
|
|
405
|
+
if (belowAcIds.length > 0) {
|
|
406
|
+
stderr.write(`finalise refused: EVAL below threshold on AC(s) ${belowAcIds.join(', ')}; investigate the run record or --ship-without-eval "reason"\n`);
|
|
407
|
+
}
|
|
408
|
+
stderr.write(summariseReport(passLoaded.report));
|
|
409
|
+
stderr.write(`Report: ${outPath}\n`);
|
|
410
|
+
return 4;
|
|
411
|
+
}
|
|
348
412
|
const result = await updateDocument({
|
|
349
413
|
projectRoot, tree, id: fbsId, sets: [{ path: 'executionStatus', value: 'verified' }], options: {},
|
|
350
414
|
});
|
package/src/cli/help.js
CHANGED
|
@@ -15,6 +15,7 @@
|
|
|
15
15
|
|
|
16
16
|
import { HELP as BUILD_HELP } from './build.js';
|
|
17
17
|
import { HELP as COVERAGE_HELP } from './coverage.js';
|
|
18
|
+
import { HELP as AUDIT_EVAL_HELP } from './eval.js';
|
|
18
19
|
import { HELP as CREATE_HELP } from './create.js';
|
|
19
20
|
import { HELP as DELETE_HELP } from './delete.js';
|
|
20
21
|
import { HELP as DOCTOR_HELP } from './doctor.js';
|
|
@@ -237,6 +238,8 @@ Verbs:
|
|
|
237
238
|
view Live HTML tree render (long-running server).
|
|
238
239
|
coverage [scope-id] Structural coverage over the PRD -> REQ -> US ->
|
|
239
240
|
AC -> TS -> TC chain.
|
|
241
|
+
eval coverage [scope-id] EVAL coverage over nonDeterministic ACs
|
|
242
|
+
(spec rcf-eval-node-2026-09-04 section 4).
|
|
240
243
|
trace <id> Walk the graph forward, back, or both from a node.
|
|
241
244
|
impact <id> Change-impact fan-out from a node.
|
|
242
245
|
|
|
@@ -299,6 +302,7 @@ export const HELP_MAP = {
|
|
|
299
302
|
coverage: COVERAGE_HELP,
|
|
300
303
|
trace: TRACE_HELP,
|
|
301
304
|
impact: IMPACT_HELP,
|
|
305
|
+
eval: AUDIT_EVAL_HELP,
|
|
302
306
|
},
|
|
303
307
|
};
|
|
304
308
|
|
package/src/core/store/ids.js
CHANGED
|
@@ -34,7 +34,10 @@
|
|
|
34
34
|
// Family membership (the single source for downstream re-exports)
|
|
35
35
|
// ---------------------------------------------------------------------------
|
|
36
36
|
|
|
37
|
-
|
|
37
|
+
// EVAL joins the prefix families in rcf-schemas 0.6.0 (evalId shape mirrors
|
|
38
|
+
// tsId: three-digit minimum with an optional lowercase kebab-slug prefix).
|
|
39
|
+
// EVAL is a peer of a Test Suite; it grades non-deterministic behaviour.
|
|
40
|
+
export const PREFIX_FAMILIES = Object.freeze(['REQ', 'US', 'PRD', 'BS', 'TAD', 'TS', 'EVAL']);
|
|
38
41
|
export const SUFFIX_FAMILIES = Object.freeze(['ADR', 'TAC', 'FBS', 'CN']);
|
|
39
42
|
export const UNNAMESPACED_FAMILIES = Object.freeze(['AC', 'TC']);
|
|
40
43
|
|
|
@@ -85,6 +88,7 @@ const FAMILY_TO_LOCATION = new Map([
|
|
|
85
88
|
['FBS', { kind: 'fbs', subdir: 'fbs', rootFile: null }],
|
|
86
89
|
['TS', { kind: 'testSuite', subdir: 'test-suites', rootFile: null }],
|
|
87
90
|
['CN', { kind: 'codeNode', subdir: 'code-nodes', rootFile: null }],
|
|
91
|
+
['EVAL', { kind: 'evalDoc', subdir: 'evals', rootFile: null }],
|
|
88
92
|
['PRD', { kind: 'prd', subdir: null, rootFile: 'prd.json' }],
|
|
89
93
|
['TAD', { kind: 'tad', subdir: null, rootFile: 'tad.json' }],
|
|
90
94
|
['BS', { kind: 'buildSequence', subdir: null, rootFile: 'build-sequence.json' }],
|
package/src/core/store/init.js
CHANGED
|
@@ -231,6 +231,10 @@ export async function initProject({ projectRoot, projectName = 'New RCF Project'
|
|
|
231
231
|
'rcf/adrs',
|
|
232
232
|
'rcf/fbs',
|
|
233
233
|
'rcf/test-suites',
|
|
234
|
+
// rcf-schemas 0.6.0: EVAL doc subdir. Empty by default; created so
|
|
235
|
+
// `rcf audit eval coverage` can find the subdir when the walker
|
|
236
|
+
// enumerates children even before any EVAL has been authored.
|
|
237
|
+
'rcf/evals',
|
|
234
238
|
];
|
|
235
239
|
for (const d of dirs) {
|
|
236
240
|
await mkdir(join(projectRoot, d), { recursive: true });
|
package/src/core/store/loader.js
CHANGED
|
@@ -30,6 +30,10 @@ export function subdirFor(kind) {
|
|
|
30
30
|
case 'testSuite': return 'test-suites';
|
|
31
31
|
// Phase 10 (X2 CodeNode bridge): Code Node document type.
|
|
32
32
|
case 'codeNode': return 'code-nodes';
|
|
33
|
+
// rcf-schemas 0.6.0 EVAL doc type. One file per EVAL under rcf/evals/.
|
|
34
|
+
// EVAL is optional in the chain; required only for ACs marked
|
|
35
|
+
// determinism: nonDeterministic.
|
|
36
|
+
case 'evalDoc': return 'evals';
|
|
33
37
|
default: return null;
|
|
34
38
|
}
|
|
35
39
|
}
|
|
@@ -28,6 +28,9 @@ import testSuiteSchema from '@stravica-ai/rcf-schemas/schemas/test-suite.schema.
|
|
|
28
28
|
// Phase 10 (X2 CodeNode bridge): 11th document kind, delivered in
|
|
29
29
|
// @stravica-ai/rcf-schemas@0.3.0.
|
|
30
30
|
import cnSchema from '@stravica-ai/rcf-schemas/schemas/cn.schema.json' with { type: 'json' };
|
|
31
|
+
// rcf-schemas 0.6.0: the EVAL doc type. Optional peer of a Test Suite,
|
|
32
|
+
// used to grade nonDeterministic acceptance criteria.
|
|
33
|
+
import evalSchema from '@stravica-ai/rcf-schemas/schemas/eval.schema.json' with { type: 'json' };
|
|
31
34
|
|
|
32
35
|
import { rcfError } from '../errors/index.js';
|
|
33
36
|
|
|
@@ -39,7 +42,7 @@ import { rcfError } from '../errors/index.js';
|
|
|
39
42
|
// schema, unmodified.
|
|
40
43
|
|
|
41
44
|
/**
|
|
42
|
-
* @typedef {('manifest'|'prd'|'req'|'userStory'|'tad'|'tac'|'adr'|'buildSequence'|'fbs'|'testSuite'|'codeNode')} DocKind
|
|
45
|
+
* @typedef {('manifest'|'prd'|'req'|'userStory'|'tad'|'tac'|'adr'|'buildSequence'|'fbs'|'testSuite'|'codeNode'|'evalDoc')} DocKind
|
|
43
46
|
*/
|
|
44
47
|
|
|
45
48
|
const SCHEMAS = {
|
|
@@ -55,6 +58,8 @@ const SCHEMAS = {
|
|
|
55
58
|
testSuite: testSuiteSchema,
|
|
56
59
|
// Phase 10: Code Node.
|
|
57
60
|
codeNode: cnSchema,
|
|
61
|
+
// rcf-schemas 0.6.0: EVAL doc.
|
|
62
|
+
evalDoc: evalSchema,
|
|
58
63
|
};
|
|
59
64
|
|
|
60
65
|
const ID_FIELD = {
|
|
@@ -72,6 +77,8 @@ const ID_FIELD = {
|
|
|
72
77
|
testSuite: 'id',
|
|
73
78
|
// Phase 10: Code Node.
|
|
74
79
|
codeNode: 'cnId',
|
|
80
|
+
// rcf-schemas 0.6.0: EVAL doc uses the plain `id` field (mirrors TS).
|
|
81
|
+
evalDoc: 'id',
|
|
75
82
|
};
|
|
76
83
|
|
|
77
84
|
let cachedAjv = null;
|
package/src/core/store/walker.js
CHANGED
|
@@ -87,7 +87,9 @@ function idFromFilenameStem(stem) {
|
|
|
87
87
|
// Phase 10 (X2 CodeNode bridge): 'codeNode' appended. The load-then-invert
|
|
88
88
|
// engine treats it exactly like any other child kind - extending the graph
|
|
89
89
|
// into code is additive, not a rewrite (PoC-proven, poc/codenode-bridge).
|
|
90
|
-
|
|
90
|
+
// rcf-schemas 0.6.0: 'evalDoc' appended. EVAL is a peer of TS; it grades
|
|
91
|
+
// non-deterministic ACs and rides the same load-then-invert engine.
|
|
92
|
+
const CHILD_KINDS = ['req', 'userStory', 'tac', 'adr', 'fbs', 'testSuite', 'codeNode', 'evalDoc'];
|
|
91
93
|
|
|
92
94
|
const ID_FIELD_BY_KIND = {
|
|
93
95
|
prd: 'prdId',
|
|
@@ -102,6 +104,8 @@ const ID_FIELD_BY_KIND = {
|
|
|
102
104
|
testSuite: 'id',
|
|
103
105
|
// Phase 10: Code Node.
|
|
104
106
|
codeNode: 'cnId',
|
|
107
|
+
// rcf-schemas 0.6.0: EVAL doc.
|
|
108
|
+
evalDoc: 'id',
|
|
105
109
|
};
|
|
106
110
|
|
|
107
111
|
function idOfDoc(doc, kind) {
|
|
@@ -125,6 +129,9 @@ function newTree() {
|
|
|
125
129
|
testSuites: [],
|
|
126
130
|
// Phase 10 (X2 CodeNode bridge): Code Nodes.
|
|
127
131
|
codeNodes: [],
|
|
132
|
+
// rcf-schemas 0.6.0: EVAL docs. A peer of testSuites; walked and
|
|
133
|
+
// integrity-checked like every other child kind.
|
|
134
|
+
evals: [],
|
|
128
135
|
byId: new Map(),
|
|
129
136
|
rawById: new Map(),
|
|
130
137
|
kindById: new Map(),
|
|
@@ -146,6 +153,10 @@ function newTree() {
|
|
|
146
153
|
// fbsByAcId / dependentsByFbsId for the code layer.
|
|
147
154
|
cnByAcId: new Map(),
|
|
148
155
|
dependentsByCnId: new Map(),
|
|
156
|
+
// rcf-schemas 0.6.0: EVAL -> AC inversion. Key: acId, value: EVAL ids
|
|
157
|
+
// whose acIds[] names that AC. Consumed by `rcf audit eval coverage`
|
|
158
|
+
// to answer "does this AC have a resolving EVAL?" in constant time.
|
|
159
|
+
evalByAcId: new Map(),
|
|
149
160
|
};
|
|
150
161
|
}
|
|
151
162
|
|
|
@@ -174,6 +185,8 @@ function recordDoc(tree, id, doc, raw, kind) {
|
|
|
174
185
|
case 'testSuite': tree.testSuites.push(doc); break;
|
|
175
186
|
// Phase 10: Code Node.
|
|
176
187
|
case 'codeNode': tree.codeNodes.push(doc); break;
|
|
188
|
+
// rcf-schemas 0.6.0: EVAL doc.
|
|
189
|
+
case 'evalDoc': tree.evals.push(doc); break;
|
|
177
190
|
default: break;
|
|
178
191
|
}
|
|
179
192
|
}
|
|
@@ -315,6 +328,8 @@ export async function walkTree({ projectRoot }) {
|
|
|
315
328
|
tree.fbsItems = sortById(tree.fbsItems, 'fbsId');
|
|
316
329
|
tree.testSuites = sortById(tree.testSuites, 'id');
|
|
317
330
|
tree.codeNodes = sortById(tree.codeNodes, 'cnId'); // Phase 10
|
|
331
|
+
// rcf-schemas 0.6.0: EVAL doc uses `id` field (mirrors TS).
|
|
332
|
+
tree.evals = sortById(tree.evals, 'id');
|
|
318
333
|
|
|
319
334
|
// Referential integrity + graph inversion.
|
|
320
335
|
invertGraph(tree);
|
|
@@ -416,11 +431,21 @@ function invertGraph(tree) {
|
|
|
416
431
|
}
|
|
417
432
|
}
|
|
418
433
|
|
|
434
|
+
// rcf-schemas 0.6.0: invert EVAL edges.
|
|
435
|
+
// EVAL.usId -> US (parent-child)
|
|
436
|
+
// EVAL.acIds -> evalByAcId (keyed on AC, value = EVAL grading it)
|
|
437
|
+
for (const evalDoc of tree.evals) {
|
|
438
|
+
if (isKind(evalDoc.usId, 'userStory')) linkParent(evalDoc.id, evalDoc.usId);
|
|
439
|
+
for (const acId of evalDoc.acIds ?? []) {
|
|
440
|
+
if (acIds.has(acId)) pushToMap(tree.evalByAcId, acId, evalDoc.id);
|
|
441
|
+
}
|
|
442
|
+
}
|
|
443
|
+
|
|
419
444
|
// Sort children lists deterministically.
|
|
420
445
|
for (const [k, list] of tree.childrenByParent) {
|
|
421
446
|
tree.childrenByParent.set(k, [...list].sort());
|
|
422
447
|
}
|
|
423
|
-
for (const map of [tree.fbsByAcId, tree.dependentsByFbsId, tree.tsByAcId, tree.usByTacId, tree.cnByAcId, tree.dependentsByCnId]) {
|
|
448
|
+
for (const map of [tree.fbsByAcId, tree.dependentsByFbsId, tree.tsByAcId, tree.usByTacId, tree.cnByAcId, tree.dependentsByCnId, tree.evalByAcId]) {
|
|
424
449
|
for (const [k, list] of map) map.set(k, [...list].sort());
|
|
425
450
|
}
|
|
426
451
|
}
|
|
@@ -519,6 +544,8 @@ export function simulateWriteErrors({ tree, preErrors = [], upserts = [], delete
|
|
|
519
544
|
post.fbsItems = sortById(post.fbsItems, 'fbsId');
|
|
520
545
|
post.testSuites = sortById(post.testSuites, 'id');
|
|
521
546
|
post.codeNodes = sortById(post.codeNodes, 'cnId'); // Phase 10
|
|
547
|
+
// rcf-schemas 0.6.0: EVAL doc.
|
|
548
|
+
post.evals = sortById(post.evals, 'id');
|
|
522
549
|
invertGraph(post);
|
|
523
550
|
collectBrokenReferences(post, errors);
|
|
524
551
|
// w-2026-07-28-017: uniqueness is part of the post-write gate, so a
|
|
@@ -953,4 +980,46 @@ function collectBrokenReferences(tree, errors) {
|
|
|
953
980
|
});
|
|
954
981
|
}
|
|
955
982
|
}
|
|
983
|
+
|
|
984
|
+
// rcf-schemas 0.6.0: EVAL doc cross-link integrity. Mirrors the TS
|
|
985
|
+
// parent + acIds shape (usId -> parent US; every acId -> a known AC).
|
|
986
|
+
// Cross-US EVAL bindings are refused at v1: an EVAL only grades ACs
|
|
987
|
+
// that live on its parent US, matching the TS scoping rule.
|
|
988
|
+
for (const evalDoc of tree.evals) {
|
|
989
|
+
check({
|
|
990
|
+
docId: evalDoc.id,
|
|
991
|
+
docKind: 'evalDoc',
|
|
992
|
+
fromField: 'usId',
|
|
993
|
+
targetId: evalDoc.usId,
|
|
994
|
+
expectedKind: 'userStory',
|
|
995
|
+
filePath: `rcf/evals/${(evalDoc.id ?? '').toLowerCase()}.json`,
|
|
996
|
+
message: `EVAL ${evalDoc.id} references unknown US ${evalDoc.usId}`,
|
|
997
|
+
});
|
|
998
|
+
const parentUs = tree.userStories.find((u) => u.usId === evalDoc.usId);
|
|
999
|
+
const parentAcIds = new Set(
|
|
1000
|
+
(parentUs?.acceptanceCriteria ?? []).map((ac) => ac?.id).filter(Boolean),
|
|
1001
|
+
);
|
|
1002
|
+
for (const [i, acId] of (evalDoc.acIds ?? []).entries()) {
|
|
1003
|
+
if (!acIds.has(acId)) {
|
|
1004
|
+
errors.push(rcfError({
|
|
1005
|
+
kind: 'brokenReference',
|
|
1006
|
+
message: `EVAL ${evalDoc.id} references unknown acceptance criterion ${acId}`,
|
|
1007
|
+
documentId: evalDoc.id,
|
|
1008
|
+
filePath: `rcf/evals/${(evalDoc.id ?? '').toLowerCase()}.json`,
|
|
1009
|
+
field: `acIds[${i}]`,
|
|
1010
|
+
rule: 'resolveTo:ac',
|
|
1011
|
+
}));
|
|
1012
|
+
tree.brokenIds.add(acId);
|
|
1013
|
+
} else if (parentUs && !parentAcIds.has(acId)) {
|
|
1014
|
+
errors.push(rcfError({
|
|
1015
|
+
kind: 'brokenReference',
|
|
1016
|
+
message: `EVAL ${evalDoc.id} lists AC ${acId} that lives outside its parent US ${evalDoc.usId} (cross-US EVAL bindings are not permitted)`,
|
|
1017
|
+
documentId: evalDoc.id,
|
|
1018
|
+
filePath: `rcf/evals/${(evalDoc.id ?? '').toLowerCase()}.json`,
|
|
1019
|
+
field: `acIds[${i}]`,
|
|
1020
|
+
rule: 'evalAcIdsScopedToParentUs',
|
|
1021
|
+
}));
|
|
1022
|
+
}
|
|
1023
|
+
}
|
|
1024
|
+
}
|
|
956
1025
|
}
|
package/src/core/store/writer.js
CHANGED
|
@@ -862,6 +862,10 @@ async function createInlineAc({ projectRoot, tree, options, body, walkErrors = [
|
|
|
862
862
|
...(body?.given !== undefined ? { given: body.given } : {}),
|
|
863
863
|
...(body?.when !== undefined ? { when: body.when } : {}),
|
|
864
864
|
...(body?.then !== undefined ? { then: body.then } : {}),
|
|
865
|
+
// rcf-eval-node spec 2026-09-04 section 2.2: authored determinism
|
|
866
|
+
// survives write. Absence is treated as 'deterministic' by every
|
|
867
|
+
// consumer, so we only serialise when the CLI passed a value.
|
|
868
|
+
...(body?.determinism !== undefined ? { determinism: body.determinism } : {}),
|
|
865
869
|
};
|
|
866
870
|
const nextAcs = replacingPhantom
|
|
867
871
|
? currentAcs.map((ac, i) => (i === seedIndex ? acEntry : ac))
|