vouchington-tooling 0.85.1 → 0.85.2
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/dist/cli/commands/final-review.d.mts +8 -0
- package/dist/cli/commands/final-review.mjs +70 -0
- package/dist/cli/index.mjs +2 -0
- package/dist/cli/parse-forwarded-commands.d.mts +1 -1
- package/dist/cli/parse-forwarded-commands.mjs +1 -0
- package/dist/cli/parse.d.mts +3 -0
- package/dist/cli/usage-text-final-review.d.mts +2 -0
- package/dist/cli/usage-text-final-review.mjs +22 -0
- package/dist/cli/usage-text-policy.d.mts +2 -2
- package/dist/cli/usage-text-policy.mjs +3 -2
- package/dist/cli/usage-text.d.mts +1 -1
- package/dist/final-review/benchmark/cases.d.mts +23 -0
- package/dist/final-review/benchmark/cases.mjs +89 -0
- package/dist/final-review/benchmark/cli.d.mts +12 -0
- package/dist/final-review/benchmark/cli.mjs +144 -0
- package/dist/final-review/benchmark/judge-all.d.mts +32 -0
- package/dist/final-review/benchmark/judge-all.mjs +50 -0
- package/dist/final-review/benchmark/judge.d.mts +18 -0
- package/dist/final-review/benchmark/judge.mjs +86 -0
- package/dist/final-review/benchmark/runner.d.mts +60 -0
- package/dist/final-review/benchmark/runner.mjs +85 -0
- package/dist/final-review/benchmark/score.d.mts +33 -0
- package/dist/final-review/benchmark/score.mjs +84 -0
- package/dist/final-review/benchmark/snapshot.d.mts +20 -0
- package/dist/final-review/benchmark/snapshot.mjs +107 -0
- package/dist/final-review/benchmark/verifier-check.d.mts +21 -0
- package/dist/final-review/benchmark/verifier-check.mjs +93 -0
- package/dist/final-review/benchmark/verifier-cli.d.mts +6 -0
- package/dist/final-review/benchmark/verifier-cli.mjs +141 -0
- package/dist/final-review/benchmark/verifier-grade.d.mts +42 -0
- package/dist/final-review/benchmark/verifier-grade.mjs +76 -0
- package/dist/final-review/benchmark/verifier-input.d.mts +21 -0
- package/dist/final-review/benchmark/verifier-input.mjs +47 -0
- package/dist/final-review/benchmark/verifier-run.d.mts +23 -0
- package/dist/final-review/benchmark/verifier-run.mjs +108 -0
- package/dist/final-review/benchmark/verifier-score.d.mts +39 -0
- package/dist/final-review/benchmark/verifier-score.mjs +70 -0
- package/dist/final-review/build.d.mts +21 -0
- package/dist/final-review/build.mjs +91 -0
- package/dist/final-review/diff-files.d.mts +16 -0
- package/dist/final-review/diff-files.mjs +44 -0
- package/dist/final-review/finding.d.mts +32 -0
- package/dist/final-review/finding.mjs +101 -0
- package/dist/final-review/github.d.mts +21 -0
- package/dist/final-review/github.mjs +105 -0
- package/dist/final-review/index.d.mts +11 -0
- package/dist/final-review/index.mjs +6 -0
- package/dist/final-review/numbered-diff.d.mts +9 -0
- package/dist/final-review/numbered-diff.mjs +52 -0
- package/dist/final-review/post.d.mts +33 -0
- package/dist/final-review/post.mjs +63 -0
- package/dist/final-review/render.d.mts +17 -0
- package/dist/final-review/render.mjs +81 -0
- package/dist/gh-cli/exec.mjs +3 -1
- package/dist/gha-review-payload/index.d.mts +1 -1
- package/dist/gha-review-payload/index.mjs +1 -1
- package/dist/gha-review-payload/remap.d.mts +3 -1
- package/dist/gha-review-payload/remap.mjs +3 -2
- package/package.json +4 -4
- package/scripts/__pycache__/run_bounded_process.cpython-314.pyc +0 -0
- package/skills/bounded-iteration/SKILL.md +29 -14
- package/skills/final-review/SKILL.md +95 -0
- package/skills/final-review/references/dispatch.md +40 -0
- package/skills/final-review/references/orchestrator.md +39 -0
- package/skills/final-review/references/pillars/_contract.md +61 -0
- package/skills/final-review/references/pillars/blast-radius.md +33 -0
- package/skills/final-review/references/pillars/congruence.md +25 -0
- package/skills/final-review/references/pillars/correctness.md +52 -0
- package/skills/final-review/references/pillars/design.md +30 -0
- package/skills/final-review/references/pillars/history.md +26 -0
- package/skills/final-review/references/pillars/rules.md +34 -0
- package/skills/final-review/references/pillars/security.md +28 -0
- package/skills/final-review/references/verifier.md +62 -0
- package/skills/manifest.json +34 -28
- package/skills/postgres-node-performance-tuning/references/performance-patterns.md +5 -1
|
@@ -0,0 +1,8 @@
|
|
|
1
|
+
import { type FinalReviewIo, type PullTarget } from '../../final-review/index.mts';
|
|
2
|
+
export type FinalReviewDeps = {
|
|
3
|
+
createIo(target: PullTarget): FinalReviewIo;
|
|
4
|
+
readFile(path: string): string;
|
|
5
|
+
runBench(args: string[]): Promise<number>;
|
|
6
|
+
runVerifierBench(args: string[]): Promise<number>;
|
|
7
|
+
};
|
|
8
|
+
export declare function runFinalReviewCommand(args: string[], deps?: FinalReviewDeps): Promise<number>;
|
|
@@ -0,0 +1,70 @@
|
|
|
1
|
+
import { readFileSync } from 'node:fs';
|
|
2
|
+
import { parseArgs } from 'node:util';
|
|
3
|
+
import { createGhFinalReviewIo, finalReviewStatus, parseFinalReviewInput, numberDiff, postFinalReview, revokeFinalApproval, } from "../../final-review/index.mjs";
|
|
4
|
+
const defaultDeps = {
|
|
5
|
+
createIo: (target) => createGhFinalReviewIo(target),
|
|
6
|
+
readFile: (path) => readFileSync(path, 'utf8'),
|
|
7
|
+
runBench: async (args) => (await import("../../final-review/benchmark/cli.mjs")).runFinalReviewBenchCli(args),
|
|
8
|
+
runVerifierBench: async (args) => (await import("../../final-review/benchmark/verifier-cli.mjs")).runFinalReviewVerifierBenchCli(args),
|
|
9
|
+
};
|
|
10
|
+
function target(values) {
|
|
11
|
+
if (values.repo === undefined || values.pr === undefined)
|
|
12
|
+
throw new Error('--repo and --pr are required.');
|
|
13
|
+
return { repo: values.repo, pr: Number(values.pr) };
|
|
14
|
+
}
|
|
15
|
+
function print(value) {
|
|
16
|
+
process.stdout.write(`${JSON.stringify(value, null, 2)}\n`);
|
|
17
|
+
}
|
|
18
|
+
async function run(subcommand, args, deps) {
|
|
19
|
+
if (subcommand === 'bench')
|
|
20
|
+
return deps.runBench(args);
|
|
21
|
+
if (subcommand === 'bench-verifier')
|
|
22
|
+
return deps.runVerifierBench(args);
|
|
23
|
+
const { values } = parseArgs({
|
|
24
|
+
args,
|
|
25
|
+
strict: true,
|
|
26
|
+
options: {
|
|
27
|
+
repo: { type: 'string' },
|
|
28
|
+
pr: { type: 'string' },
|
|
29
|
+
'head-sha': { type: 'string' },
|
|
30
|
+
input: { type: 'string' },
|
|
31
|
+
'dry-run': { type: 'boolean' },
|
|
32
|
+
},
|
|
33
|
+
});
|
|
34
|
+
const io = deps.createIo(target(values));
|
|
35
|
+
switch (subcommand) {
|
|
36
|
+
case 'post': {
|
|
37
|
+
if (values['head-sha'] === undefined || values.input === undefined) {
|
|
38
|
+
throw new Error('post needs --head-sha and --input.');
|
|
39
|
+
}
|
|
40
|
+
const input = parseFinalReviewInput(JSON.parse(deps.readFile(values.input)));
|
|
41
|
+
const outcome = await postFinalReview(io, input, values['head-sha'], {
|
|
42
|
+
dryRun: values['dry-run'] === true,
|
|
43
|
+
});
|
|
44
|
+
print(values['dry-run'] === true ? outcome : { ...outcome, reviews: outcome.reviews.length });
|
|
45
|
+
return 0;
|
|
46
|
+
}
|
|
47
|
+
case 'diff':
|
|
48
|
+
process.stdout.write(numberDiff(await io.diff()));
|
|
49
|
+
return 0;
|
|
50
|
+
case 'revoke':
|
|
51
|
+
print({ removed: await revokeFinalApproval(io) });
|
|
52
|
+
return 0;
|
|
53
|
+
case 'status': {
|
|
54
|
+
const status = await finalReviewStatus(io);
|
|
55
|
+
print(status);
|
|
56
|
+
return status.approved ? 0 : 1;
|
|
57
|
+
}
|
|
58
|
+
default:
|
|
59
|
+
throw new Error('final-review needs post, diff, revoke, status, bench, or bench-verifier.');
|
|
60
|
+
}
|
|
61
|
+
}
|
|
62
|
+
export async function runFinalReviewCommand(args, deps = defaultDeps) {
|
|
63
|
+
try {
|
|
64
|
+
return await run(args[0], args.slice(1), deps);
|
|
65
|
+
}
|
|
66
|
+
catch (error) {
|
|
67
|
+
process.stderr.write(`final-review: ${error instanceof Error ? error.message : String(error)}\n`);
|
|
68
|
+
return 2;
|
|
69
|
+
}
|
|
70
|
+
}
|
package/dist/cli/index.mjs
CHANGED
|
@@ -126,6 +126,8 @@ export function runCli(argv = process.argv) {
|
|
|
126
126
|
return runWorktreeCommand(parsed.args);
|
|
127
127
|
case 'tmux':
|
|
128
128
|
return runTmuxCommand(parsed.args);
|
|
129
|
+
case 'final-review':
|
|
130
|
+
return import("./commands/final-review.mjs").then((command) => command.runFinalReviewCommand(parsed.args));
|
|
129
131
|
case 'require-up-to-date':
|
|
130
132
|
return runRequireUpToDate(parsed);
|
|
131
133
|
case 'fresh-base':
|
|
@@ -1,4 +1,4 @@
|
|
|
1
|
-
declare const FORWARDED: readonly ["with-host-lock", "host-storage", "pnpm-install", "vitest-blob-manifest", "vitest-report-attempt", "prepare-vitest-reports", "nuget-central-version", "verify-workflow-secrets", "runner-policy", "dead-code", "swift-semantic-equal", "post-review", "stage-review-payload", "retrospective-transcript", "retrospective-facts", "agent-blackboard", "mcp", "hook", "worktree", "transient-retry", "sandbox-command-audit", "cloc", "coverage-scope-check", "trivy-policy", "docker-image-mirror", "workspace-registry-check", "storybook-browser-runner", "tmux"];
|
|
1
|
+
declare const FORWARDED: readonly ["with-host-lock", "host-storage", "pnpm-install", "vitest-blob-manifest", "vitest-report-attempt", "prepare-vitest-reports", "nuget-central-version", "verify-workflow-secrets", "runner-policy", "dead-code", "swift-semantic-equal", "post-review", "stage-review-payload", "retrospective-transcript", "retrospective-facts", "agent-blackboard", "mcp", "hook", "worktree", "transient-retry", "sandbox-command-audit", "cloc", "coverage-scope-check", "trivy-policy", "docker-image-mirror", "workspace-registry-check", "storybook-browser-runner", "tmux", "final-review"];
|
|
2
2
|
type Name = (typeof FORWARDED)[number];
|
|
3
3
|
type ForwardedCommand = {
|
|
4
4
|
[K in Name]: {
|
package/dist/cli/parse.d.mts
CHANGED
|
@@ -0,0 +1,2 @@
|
|
|
1
|
+
export declare const FINAL_REVIEW_COMMAND = " final-review Post, revoke, check, or benchmark multi-pillar final reviews\n";
|
|
2
|
+
export declare const FINAL_REVIEW_USAGE = "final-review post --repo owner/name --pr N --head-sha SHA --input findings.json [--dry-run]\nfinal-review diff --repo owner/name --pr N\nfinal-review revoke --repo owner/name --pr N\nfinal-review status --repo owner/name --pr N\nfinal-review bench --cases cases.json --prompts DIR --out DIR [--pillar NAME]... [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL]\nfinal-review bench-verifier --cases cases.json --findings-from DIR --prompts DIR --out DIR [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL] [--checker MODEL]\n post renders verified findings as one or more COMMENT reviews (verdict, reviewed SHA and effort in\n the header; inline comments where the diff allows, the rest in the body) and adds the \"final\n approval\" label on Approve or removes it otherwise. It refuses when the head moved. diff prints the pull request\n diff with base and head line numbers on every line, for reviewers to cite. revoke removes\n the label before a re-review. status exits 0 only when the newest final review approved the current\n head and the label is present. bench runs each pillar prompt against labeled cases from a git-free\n snapshot of the pre-merge head and scores recall and precision; --judge grades by claim\n with a model instead of by location. bench-verifier runs only the verifier, one call per case\n and no subagents, on the pillar findings a bench --out DIR saved (each judged-real finding is\n also planted under a second pillar), and scores real findings kept, label recall, severity\n agreement, duplicates merged, and other blocking and high findings confirmed by --checker;\n --prompts is the skill's references directory.\n\n";
|
|
@@ -0,0 +1,22 @@
|
|
|
1
|
+
export const FINAL_REVIEW_COMMAND = ` final-review Post, revoke, check, or benchmark multi-pillar final reviews
|
|
2
|
+
`;
|
|
3
|
+
export const FINAL_REVIEW_USAGE = `final-review post --repo owner/name --pr N --head-sha SHA --input findings.json [--dry-run]
|
|
4
|
+
final-review diff --repo owner/name --pr N
|
|
5
|
+
final-review revoke --repo owner/name --pr N
|
|
6
|
+
final-review status --repo owner/name --pr N
|
|
7
|
+
final-review bench --cases cases.json --prompts DIR --out DIR [--pillar NAME]... [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL]
|
|
8
|
+
final-review bench-verifier --cases cases.json --findings-from DIR --prompts DIR --out DIR [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL] [--checker MODEL]
|
|
9
|
+
post renders verified findings as one or more COMMENT reviews (verdict, reviewed SHA and effort in
|
|
10
|
+
the header; inline comments where the diff allows, the rest in the body) and adds the "final
|
|
11
|
+
approval" label on Approve or removes it otherwise. It refuses when the head moved. diff prints the pull request
|
|
12
|
+
diff with base and head line numbers on every line, for reviewers to cite. revoke removes
|
|
13
|
+
the label before a re-review. status exits 0 only when the newest final review approved the current
|
|
14
|
+
head and the label is present. bench runs each pillar prompt against labeled cases from a git-free
|
|
15
|
+
snapshot of the pre-merge head and scores recall and precision; --judge grades by claim
|
|
16
|
+
with a model instead of by location. bench-verifier runs only the verifier, one call per case
|
|
17
|
+
and no subagents, on the pillar findings a bench --out DIR saved (each judged-real finding is
|
|
18
|
+
also planted under a second pillar), and scores real findings kept, label recall, severity
|
|
19
|
+
agreement, duplicates merged, and other blocking and high findings confirmed by --checker;
|
|
20
|
+
--prompts is the skill's references directory.
|
|
21
|
+
|
|
22
|
+
`;
|
|
@@ -1,2 +1,2 @@
|
|
|
1
|
-
export declare const POLICY_COMMANDS = " coverage-scope-check Check positive-threshold coverage rules against the Vitest scope\n trivy-policy Check tracked Trivy workflow scans and the exception registry\n docker-image-mirror Require tracked Docker images to come from a Docker Hub mirror\n workspace-registry-check Check syncpack, pnpm workspace and CI path-filter registries against tracked manifests\n";
|
|
2
|
-
export declare const POLICY_USAGE = "\ncoverage-scope-check [--rules <path>] [--scope-file <json>] [--include <glob>]... [--exclude <glob>]...\n Fails (exit 1) when a git-tracked file that a coverage rules file (default .coverage-rules.yml)\n gives a patch_coverage_min above 0, and its scope does not ignore, is outside the collector's\n scope. The collector scope is the repeated --include/--exclude globs plus, optionally, the\n include/exclude string arrays of a --scope-file JSON written from the Vitest coverage config.\n At least one include glob is required.\n\ntrivy-policy [--root <dir>] [--ignore-file <path>] [--findings-exit-code-env <NAME>] [--cleanup-url-prefix <url>] [--min-gates N] [--min-sboms N] [--max-expiry-days N]\n Reads git-tracked workflows, composite actions and shell scripts and prints each violation (exit 1).\n Every Trivy package scan must be an auditable CLI run block (no action, wrapper, substitution or\n background). Vulnerability gates need --scanners vuln, --pkg-types os, --severity CRITICAL,HIGH,\n --ignore-unfixed, --skip-db-update, --quiet, table output to a file, --ignorefile <ignore-file>\n (default .trivyignore.yaml) and --exit-code \"$<NAME>\" (default TRIVY_FINDINGS_EXIT_CODE).\n CycloneDX SBOM scans must be unfiltered with --exit-code 0. At least --min-gates gates and\n --min-sboms SBOM scans (default 2 each) must exist. The tracked ignore registry needs purl-exact OS\n packages, a \"Not exploitable: ... Cleanup: <issue URL>\" statement (URL must start with\n --cleanup-url-prefix when given) and an expired_at between tomorrow and --max-expiry-days (30).\n\ndocker-image-mirror [--root <dir>] [--mirror <host>] [--allow-registry <prefix>]... [--require-digest]\n [--allow-ref-expression <text>]...\n Prints each Docker Hub image in git-tracked workflows, composite actions and Dockerfiles (exit 1):\n run, pull and create commands, containers, services, docker:// references, buildx drivers, FROM,\n COPY --from and # syntax. Images must name a registry host; --mirror (default mirror.gcr.io) is\n the host the message tells you to use. --allow-registry (repeatable) instead requires every image,\n including FROM and COPY --from, to start with one of the prefixes; a reference that expands a\n variable passes only when its literal text before the variable starts with a prefix. --require-digest\n requires @sha256: plus 64 lowercase hex characters. Stage aliases and scratch stay exempt.\n --allow-ref-expression (repeatable) is an audited escape hatch for dynamic job outputs: a reference\n whose raw text exactly equals the value is accepted. Images the same file builds locally are exempt.\n\nworkspace-registry-check [--root <dir>] [--syncpack-config <path>] [--workspace-file <path>]\n [--path-filters <path>] [--tooling-group <name>] [--trigger-only-glob <glob>] [--runtime-prefix <prefix>]\n Reconciles git-tracked package.json manifests with the syncpack \"source\" globs (default\n .syncpackrc.json), the pnpm \"packages\" globs (pnpm-workspace.yaml) and the dorny/paths-filter groups\n (.github/ci-path-filters.yml) and prints one line per violation (exit 1). The --tooling-group group\n (default tooling) must reach every manifest and wildcard workspace glob, and its --trigger-only-glob\n (default **/package.json) does not count as registering a manifest. Each group named with\n --runtime-prefix (default runtime-) must mirror the primary group of the same name.\n\n";
|
|
1
|
+
export declare const POLICY_COMMANDS = " coverage-scope-check Check positive-threshold coverage rules against the Vitest scope\n trivy-policy Check tracked Trivy workflow scans and the exception registry\n docker-image-mirror Require tracked Docker images to come from a Docker Hub mirror\n workspace-registry-check Check syncpack, pnpm workspace and CI path-filter registries against tracked manifests\n final-review Post, revoke, check, or benchmark multi-pillar final reviews\n";
|
|
2
|
+
export declare const POLICY_USAGE = "\nfinal-review post --repo owner/name --pr N --head-sha SHA --input findings.json [--dry-run]\nfinal-review diff --repo owner/name --pr N\nfinal-review revoke --repo owner/name --pr N\nfinal-review status --repo owner/name --pr N\nfinal-review bench --cases cases.json --prompts DIR --out DIR [--pillar NAME]... [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL]\nfinal-review bench-verifier --cases cases.json --findings-from DIR --prompts DIR --out DIR [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL] [--checker MODEL]\n post renders verified findings as one or more COMMENT reviews (verdict, reviewed SHA and effort in\n the header; inline comments where the diff allows, the rest in the body) and adds the \"final\n approval\" label on Approve or removes it otherwise. It refuses when the head moved. diff prints the pull request\n diff with base and head line numbers on every line, for reviewers to cite. revoke removes\n the label before a re-review. status exits 0 only when the newest final review approved the current\n head and the label is present. bench runs each pillar prompt against labeled cases from a git-free\n snapshot of the pre-merge head and scores recall and precision; --judge grades by claim\n with a model instead of by location. bench-verifier runs only the verifier, one call per case\n and no subagents, on the pillar findings a bench --out DIR saved (each judged-real finding is\n also planted under a second pillar), and scores real findings kept, label recall, severity\n agreement, duplicates merged, and other blocking and high findings confirmed by --checker;\n --prompts is the skill's references directory.\n\ncoverage-scope-check [--rules <path>] [--scope-file <json>] [--include <glob>]... [--exclude <glob>]...\n Fails (exit 1) when a git-tracked file that a coverage rules file (default .coverage-rules.yml)\n gives a patch_coverage_min above 0, and its scope does not ignore, is outside the collector's\n scope. The collector scope is the repeated --include/--exclude globs plus, optionally, the\n include/exclude string arrays of a --scope-file JSON written from the Vitest coverage config.\n At least one include glob is required.\n\ntrivy-policy [--root <dir>] [--ignore-file <path>] [--findings-exit-code-env <NAME>] [--cleanup-url-prefix <url>] [--min-gates N] [--min-sboms N] [--max-expiry-days N]\n Reads git-tracked workflows, composite actions and shell scripts and prints each violation (exit 1).\n Every Trivy package scan must be an auditable CLI run block (no action, wrapper, substitution or\n background). Vulnerability gates need --scanners vuln, --pkg-types os, --severity CRITICAL,HIGH,\n --ignore-unfixed, --skip-db-update, --quiet, table output to a file, --ignorefile <ignore-file>\n (default .trivyignore.yaml) and --exit-code \"$<NAME>\" (default TRIVY_FINDINGS_EXIT_CODE).\n CycloneDX SBOM scans must be unfiltered with --exit-code 0. At least --min-gates gates and\n --min-sboms SBOM scans (default 2 each) must exist. The tracked ignore registry needs purl-exact OS\n packages, a \"Not exploitable: ... Cleanup: <issue URL>\" statement (URL must start with\n --cleanup-url-prefix when given) and an expired_at between tomorrow and --max-expiry-days (30).\n\ndocker-image-mirror [--root <dir>] [--mirror <host>] [--allow-registry <prefix>]... [--require-digest]\n [--allow-ref-expression <text>]...\n Prints each Docker Hub image in git-tracked workflows, composite actions and Dockerfiles (exit 1):\n run, pull and create commands, containers, services, docker:// references, buildx drivers, FROM,\n COPY --from and # syntax. Images must name a registry host; --mirror (default mirror.gcr.io) is\n the host the message tells you to use. --allow-registry (repeatable) instead requires every image,\n including FROM and COPY --from, to start with one of the prefixes; a reference that expands a\n variable passes only when its literal text before the variable starts with a prefix. --require-digest\n requires @sha256: plus 64 lowercase hex characters. Stage aliases and scratch stay exempt.\n --allow-ref-expression (repeatable) is an audited escape hatch for dynamic job outputs: a reference\n whose raw text exactly equals the value is accepted. Images the same file builds locally are exempt.\n\nworkspace-registry-check [--root <dir>] [--syncpack-config <path>] [--workspace-file <path>]\n [--path-filters <path>] [--tooling-group <name>] [--trigger-only-glob <glob>] [--runtime-prefix <prefix>]\n Reconciles git-tracked package.json manifests with the syncpack \"source\" globs (default\n .syncpackrc.json), the pnpm \"packages\" globs (pnpm-workspace.yaml) and the dorny/paths-filter groups\n (.github/ci-path-filters.yml) and prints one line per violation (exit 1). The --tooling-group group\n (default tooling) must reach every manifest and wildcard workspace glob, and its --trigger-only-glob\n (default **/package.json) does not count as registering a manifest. Each group named with\n --runtime-prefix (default runtime-) must mirror the primary group of the same name.\n\n";
|
|
@@ -1,11 +1,12 @@
|
|
|
1
1
|
import { COVERAGE_SCOPE_CHECK_USAGE } from "./usage-text-coverage-scope.mjs";
|
|
2
|
+
import { FINAL_REVIEW_COMMAND, FINAL_REVIEW_USAGE } from "./usage-text-final-review.mjs";
|
|
2
3
|
export const POLICY_COMMANDS = ` coverage-scope-check Check positive-threshold coverage rules against the Vitest scope
|
|
3
4
|
trivy-policy Check tracked Trivy workflow scans and the exception registry
|
|
4
5
|
docker-image-mirror Require tracked Docker images to come from a Docker Hub mirror
|
|
5
6
|
workspace-registry-check Check syncpack, pnpm workspace and CI path-filter registries against tracked manifests
|
|
6
|
-
`;
|
|
7
|
+
${FINAL_REVIEW_COMMAND}`;
|
|
7
8
|
export const POLICY_USAGE = `
|
|
8
|
-
${COVERAGE_SCOPE_CHECK_USAGE}
|
|
9
|
+
${FINAL_REVIEW_USAGE}${COVERAGE_SCOPE_CHECK_USAGE}
|
|
9
10
|
trivy-policy [--root <dir>] [--ignore-file <path>] [--findings-exit-code-env <NAME>] [--cleanup-url-prefix <url>] [--min-gates N] [--min-sboms N] [--max-expiry-days N]
|
|
10
11
|
Reads git-tracked workflows, composite actions and shell scripts and prints each violation (exit 1).
|
|
11
12
|
Every Trivy package scan must be an auditable CLI run block (no action, wrapper, substitution or
|
|
@@ -1 +1 @@
|
|
|
1
|
-
export declare const USAGE = "Usage: vouchington <command> [options]\n\nCommands:\n runner-port-policy Print or validate a runner port policy\n with-host-lock Run a command under a host-wide lock\n host-storage Check free space on caller-supplied paths\n gha-runtime-audit Audit successful GitHub Actions job runtimes\n require-up-to-date Require HEAD to include a fetched remote branch\n fresh-base Classify the worktree against a remote base branch\n gitleaks-directory-scan Scan a directory with Gitleaks\n ast-grep-examples Run AST-grep rule examples\n ast-grep-pack Print shipped unconditional AST-grep pack paths as JSON\n gha-workspace-policy Check GitHub Actions workspace safety policy\n gha-output Write a collision-safe multiline GITHUB_OUTPUT record\n gha-needs-results Fail if required GitHub Actions job results failed\n download-with-diagnostics Download a URL and report HTTP status on failure\n download-optional-run-artifacts Download optional artifacts from the current run\n host-pressure-diagnostics Print a bounded host memory/OOM/PSI snapshot\n allocate-browser-safe-ports Allocate Fetch-safe localhost ports\n diagnose-port-collision Capture bounded localhost port diagnostics\n prepare-trivy-db Download the Trivy vulnerability database\n gha-artifacts-cleanup Delete classified GitHub Actions artifacts\n agent-issue-labels Map changed paths to labels; create missing repository labels\n http-origin Validate an optional HTTP(S) origin\n vitest-blob-manifest Stamp a vitest-blob-manifest:v1 identity file\n vitest-report-attempt Write or read a Vitest report-attempt marker\n prepare-vitest-reports Validate and select Vitest report JSON files\n pnpm-install Install a pnpm workspace with retry and release-age fail-fast\n check-cache-size Measure a path and decide whether to save a GHA cache\n make-shard-matrix Emit a [1..N] GitHub Actions shard matrix\n load-runner-env Overlay a runner env file onto GITHUB_ENV with injection guards\n clean-workspace Reset a persistent-runner workspace with a fork-PR trust gate\n install-github-release Download a checksum-verified GitHub Release binary\n run-with-timeout Run a command with GNU timeout or a Perl fallback\n lint-links Two-pass lychee: internal links fail, external warn\n materialize-pr-context Dump PR title/body/files/diff/comments and #N crawl\n wait-for-apt-locks Wait until apt/dpkg lock files are free\n install-playwright-chromium-arm64 Install Playwright Chromium from browsers.json\n ghcr-package-retention Delete old GHCR package versions (KEEP_MIN or --main-history)\n harness-admission-lane Compute a GITHUB_RUN_ID admission lane for fleet fan-out\n harness-assert-gates Fail if any named HARNESS_*_ENABLED gate is enabled\n gha-collaborator-trust Classify authors as write-access collaborators or not\n nuget-central-version Validate a Directory.Packages.props PackageVersion delta\n verify-workflow-secrets Check a secret inventory against live repository and environment secrets\n runner-policy Check workflow runner labels and jobs against a config\n dead-code Gate unused code against a committed jscpd baseline\n swift-semantic-equal Compare Swift sources ignoring comments and whitespace\n post-review Post one COMMENT review from a staged payload file\n stage-review-payload Validate a review payload file into a staging directory\n retrospective-transcript Format facts from Claude-compatible, Codex, or Grok transcripts\n link-skill Link one packaged skill into an explicit consumer directory\n retrospective-facts Gather immutable facts for a retrospective\n agent-blackboard Probe an Agent Blackboard deployment, classify distill sessions, manage snapshots\n mcp Serve the agent-blackboard journal tools over stdio (MCP)\n hook Run a repository policy module as a pre/post tool-use hook\n worktree Reclaim finished git worktrees (dry run unless --yes)\n tmux Run configured dev windows in tmux; name panes; agent reminders\n story-coverage Check component exports are imported by stories\n transient-retry Rerun a failed GitHub Actions run when an injected rule matches\n sandbox-command-audit Audit agent transcripts for sandbox escalations and denials\n cloc Count tracked lines of code by category and service with scc\n coverage-scope-check Check positive-threshold coverage rules against the Vitest scope\n trivy-policy Check tracked Trivy workflow scans and the exception registry\n docker-image-mirror Require tracked Docker images to come from a Docker Hub mirror\n workspace-registry-check Check syncpack, pnpm workspace and CI path-filter registries against tracked manifests\n storybook-browser-runner Run the Storybook Vitest browser project with cache, retry and watchdog\n\nOptions:\n -h, --help Show this help\n -v, --version Print the package version\n\nrunner-port-policy\n (no args) Print the shipped policy as JSON\n --file <path> Validate and print a policy file\n --reserved <port> Print true if the port is reserved\n\nwith-host-lock\n --name <family>\n [--slots <n>]\n --timeout-seconds <n>\n [--command-timeout-seconds <n>]\n [--failure-diagnostics <absolute-script>]\n [--on-acquire-timeout fail|run-unlocked]\n -- <command> [args...]\n\nhost-storage --path <path> [--path <path>...] [--minimum-free-bytes <bytes>] [--pnpm-store] [--docker-root] [--docker-desktop-path <path>] [--discovery-timeout-ms <ms>]\n\ngha-runtime-audit\n [--repository owner/name] Default GITHUB_REPOSITORY\n [--branch main]\n [--median-threshold-floor <seconds>] Off unless set. Must be below the ceiling\n [--median-threshold-ceiling <seconds>] Default 360. Flags a five-sample median above this\n --pr-workflow <name|/regex/> Repeatable\n --push-workflow <name|/regex/> Repeatable\n\nrequire-up-to-date --remote <name> --branch <name>\nfresh-base [--remote <name>] [--branch <name>] [--skip-fetch]\ngitleaks-directory-scan --config <path> [--directory <path>]\nast-grep-examples --rules <directory> --config <path>\nast-grep-pack\ngha-workspace-policy [--root <directory>] [--workflow-directory <directory>] [--action-directory <directory>]\n\ngha-output <name>\ngha-needs-results [label]\ndownload-with-diagnostics <url> <destination> [-- curl-args...]\ndownload-optional-run-artifacts (--name <name>... | --pattern <pattern>) --dir <directory>\nhost-pressure-diagnostics\nallocate-browser-safe-ports <count> [--policy path] [--forbidden-ports path]\ndiagnose-port-collision [--ports \"2200 2216\"] [--output-dir PATH]\nprepare-trivy-db\ngha-artifacts-cleanup run --run-id <id> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\ngha-artifacts-cleanup sweep --older-than-hours <n> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\nagent-issue-labels labels-from-paths [--labeler <path>] <path>...\nagent-issue-labels ensure-labels [--repo owner/name] [--policy-file json] [--update] <label>...\nhttp-origin [--field NAME] [value]\nvitest-blob-manifest <suite> [reports-directory]\nvitest-report-attempt <write DIRECTORY SUITE|read ROOT>\nprepare-vitest-reports [primary-directory] [fallback-directory] [output-directory]\npnpm-install --runner-lifecycle persistent|ephemeral|ephemeral-full --install-scripts true|false\ncheck-cache-size <path> <max-bytes> <label>\nmake-shard-matrix <total>\nload-runner-env\nclean-workspace\ninstall-github-release --repo owner/name --version X --asset 'name-{platform}.tar.gz' --bin name [--tag-prefix PREFIX] [--expected-sha256 SHA256] [--no-checksum] [--checksums-asset NAME] [--version-flag FLAG] [--bin-dir DIR]\nrun-with-timeout <timeout-seconds> <kill-after-seconds> <command...>\nlint-links [--offline] [--config PATH] [--glob PATTERN] [files...]\nmaterialize-pr-context\nwait-for-apt-locks\ninstall-playwright-chromium-arm64 [name:archive...]\nghcr-package-retention [--main-history] [--apply|--dry-run] [<url-encoded-package>...]\nharness-admission-lane <lanes>\nharness-assert-gates <gate>...\ngha-collaborator-trust <owner/repo> [--allow-bot <login>]... < authors.json\nnuget-central-version <trusted-props> <candidate-props> <metadata-json> <output-props>\nverify-workflow-secrets [--repo OWNER/NAME] [--confirm NAME,...] [--inventory MODULE] [--export NAME] [--branch NAME]\nrunner-policy [--config FILE] [--root DIR]\ndead-code --baseline PATH [--seed|--update] [--config PATH] [--binary PATH] [--repo-root PATH] [--entry GLOB]... [--snapshot-ignore GLOB]... [--exclude GLOB]...\nswift-semantic-equal <base> <head> <file.swift>\npost-review\nstage-review-payload optional|required <source> <destination>\nretrospective-transcript [--session-id ID] [--jsonl PATH] [--projects-dir PATH] [--codex-sessions-dir PATH] [--grok-sessions-dir PATH]\nlink-skill <name> --source-root <skills-dir> --target-root <consumer-skills-dir> Link a packaged or repository-local skill\nretrospective-facts (--pr NUMBER | --branch NAME | --no-pr) [--repo OWNER/NAME] [--raw]\nagent-blackboard probe\nagent-blackboard session-start [--skip-env NAME] [--setup-hint TEXT] [--docs-path PATH] [--restore-hint TEXT] [--timeout-ms N] (SessionStart hook; payload on stdin)\nagent-blackboard distill classify <partition.jsonl> --retro-cutoff ISO8601 --session-cutoff ISO8601\nagent-blackboard snapshot partition --snapshot PATH --checksum SHA256 --counts '{\"sessions\":N,\"entries\":N,\"records\":N,\"bytes\":N}'\nagent-blackboard snapshot cleanup [--snapshot PATH] [--partition-directory PATH --receipt JSON]\nmcp\n (no args) Serve journal_append, journal_entries, outbox_status, outbox_flush,\n session_ensure, retrospective_check, retrospective_save,\n pr_validate, pr_create, pr_update,\n snapshot_export, snapshot_partition, snapshot_cleanup, and session_archive over stdio.\n Needs the optional @modelcontextprotocol/sdk peer (and zod).\n Launch it from inside the repository's git worktree.\n\nhook pre-tool-use|post-tool-use --harness claude|codex|cursor|grok --policy <module> [--root <dir>] [--automation-env NAME]...\n Reads the harness hook payload on stdin. <module> default-exports a policy object (or a function\n of the hook context returning one) with a command rule table, post-tool-use checkers, and an\n optional reminder. pre-tool-use exits 2 to block, and also when the policy cannot load.\n post-tool-use is advisory and always exits 0. --root defaults to the working directory.\n Each --automation-env NAME (repeatable) marks the run unattended when that environment variable\n is set, non-empty, and not 0 or false; CI markers and claude -p are always unattended.\n\nworktree reclaim [--yes] [--json] [--max-worktrees N] [--timeout-ms N] [--pr-limit N] [--after PATH]\n Dry run by default. Removes clean, fully pushed worktrees whose same-repository PR for the\n branch is merged or closed at exactly HEAD, using git worktree remove (never forced).\n Skips the main checkout, the current worktree, and locked, bare or detached worktrees.\n\ntransient-retry --rules <module> [--repo owner/name] [--token-env NAME] [--dry-run] <run-id>\n Loads a module exporting \"rules\" (and optionally \"limits\"), evaluates them against the failed run,\n and reruns only on a match. Exits 0 on rerun, 2 when none is warranted, 1 on error.\n\nsandbox-command-audit --rules <module> [--settings-path <file>] [--repo-root <dir>]... [--worktree-glob <glob>]... [--projects-dir <dir>] [--codex-sessions-dir <dir>] [--session-id <id>] [--limit N] [--since DAYS] [--json] [--raw]\n Scans Claude and Codex transcripts scoped to this repository (git roots plus any --worktree-glob\n layouts) and reports Bash escalations, denials and permission failures by command prefix.\n <module> exports an object (default or \"rules\") with loadPolicy({ settingsPath, cwd }), which\n returns { covers(list, commandOrPrefix) } for the allow/deny/excluded lists or { error }, and\n optionally classifyFailure(command, text, source), limits and worktreeGlobs. Transcript roots\n default to CLAUDE_CONFIG_DIR/projects and CODEX_HOME/sessions. --raw prints full command text.\n\ncloc --config <file> [--scc-bin <path>] [--json]\n Counts git-tracked lines with scc (SCC_BIN or --scc-bin, else \"scc\"), classified into\n source/tests/tooling and by service, and prints an aligned table (--json: rows). <file> is a .json\n file or a module whose default (or \"config\") export is the layout: services [{ name, match }],\n defaultService, categories, testPatterns, toolingPatterns, countedRoots, countedRootFiles,\n excludedDirs, ignoredDirs, sccBin, missingSccHint.\n\ncoverage-scope-check [--rules <path>] [--scope-file <json>] [--include <glob>]... [--exclude <glob>]...\n Fails (exit 1) when a git-tracked file that a coverage rules file (default .coverage-rules.yml)\n gives a patch_coverage_min above 0, and its scope does not ignore, is outside the collector's\n scope. The collector scope is the repeated --include/--exclude globs plus, optionally, the\n include/exclude string arrays of a --scope-file JSON written from the Vitest coverage config.\n At least one include glob is required.\n\ntrivy-policy [--root <dir>] [--ignore-file <path>] [--findings-exit-code-env <NAME>] [--cleanup-url-prefix <url>] [--min-gates N] [--min-sboms N] [--max-expiry-days N]\n Reads git-tracked workflows, composite actions and shell scripts and prints each violation (exit 1).\n Every Trivy package scan must be an auditable CLI run block (no action, wrapper, substitution or\n background). Vulnerability gates need --scanners vuln, --pkg-types os, --severity CRITICAL,HIGH,\n --ignore-unfixed, --skip-db-update, --quiet, table output to a file, --ignorefile <ignore-file>\n (default .trivyignore.yaml) and --exit-code \"$<NAME>\" (default TRIVY_FINDINGS_EXIT_CODE).\n CycloneDX SBOM scans must be unfiltered with --exit-code 0. At least --min-gates gates and\n --min-sboms SBOM scans (default 2 each) must exist. The tracked ignore registry needs purl-exact OS\n packages, a \"Not exploitable: ... Cleanup: <issue URL>\" statement (URL must start with\n --cleanup-url-prefix when given) and an expired_at between tomorrow and --max-expiry-days (30).\n\ndocker-image-mirror [--root <dir>] [--mirror <host>] [--allow-registry <prefix>]... [--require-digest]\n [--allow-ref-expression <text>]...\n Prints each Docker Hub image in git-tracked workflows, composite actions and Dockerfiles (exit 1):\n run, pull and create commands, containers, services, docker:// references, buildx drivers, FROM,\n COPY --from and # syntax. Images must name a registry host; --mirror (default mirror.gcr.io) is\n the host the message tells you to use. --allow-registry (repeatable) instead requires every image,\n including FROM and COPY --from, to start with one of the prefixes; a reference that expands a\n variable passes only when its literal text before the variable starts with a prefix. --require-digest\n requires @sha256: plus 64 lowercase hex characters. Stage aliases and scratch stay exempt.\n --allow-ref-expression (repeatable) is an audited escape hatch for dynamic job outputs: a reference\n whose raw text exactly equals the value is accepted. Images the same file builds locally are exempt.\n\nworkspace-registry-check [--root <dir>] [--syncpack-config <path>] [--workspace-file <path>]\n [--path-filters <path>] [--tooling-group <name>] [--trigger-only-glob <glob>] [--runtime-prefix <prefix>]\n Reconciles git-tracked package.json manifests with the syncpack \"source\" globs (default\n .syncpackrc.json), the pnpm \"packages\" globs (pnpm-workspace.yaml) and the dorny/paths-filter groups\n (.github/ci-path-filters.yml) and prints one line per violation (exit 1). The --tooling-group group\n (default tooling) must reach every manifest and wildcard workspace glob, and its --trigger-only-glob\n (default **/package.json) does not count as registering a manifest. Each group named with\n --runtime-prefix (default runtime-) must mirror the primary group of the same name.\n\nstorybook-browser-runner --project <name> [--command <cmd>] [--result-file <path>]... [--completion-file <path>] [--coverage-arg=<arg>]... [--optimize-deps-config <path>] [--env-prefix <PREFIX>] [--progress-prefix <text>] [-- <argv>...]\n Spawns <cmd> (default pnpm) in its own process group with <argv> (default \"exec vitest run\n --project <name>\") plus the coverage args (default --coverage) and supervises it: a Vite optimizer\n cache, retries on known Vite and browser start-up flakes, and stall, hang and session-budget\n watchdogs. --result-file paths are removed before each attempt and --completion-file is the one\n whose appearance is recorded. Tuned by <PREFIX>_STARTUP_ATTEMPTS, _OPTIMIZER_STALL_MS, _HANG_MS,\n _SESSION_BUDGET_MS, _DIAGNOSTICS_FILE, _PERSISTENT_CACHE, _PERSISTENT_CACHE_DIR and _COVERAGE\n (PREFIX defaults to STORYBOOK_BROWSER). Exits with the resolved test exit code.\n\ntmux up --config <file> [--root <dir>] [--no-attach] [-v|--verbose]\ntmux name [--socket PATH --pane %ID --worktree PATH] <name> | read-title | post-tool | agent-reminder <event>\n Environment: TMUX_TARGET_BIN, TMUX_TARGET_ROOT, TMUX_NAME_COMMAND, AGENT_TMUX_SOCKET/PANE/WORKTREE.\n\nstory-coverage --root <dir> --component-root <rel> --story-root <rel> --source-extension <ext> --component-extension <ext> --marker <regex> --story-pattern <regex> [--alias <prefix>=<path>] [--exclusions <json>] [--exclusions-label <text>]\n";
|
|
1
|
+
export declare const USAGE = "Usage: vouchington <command> [options]\n\nCommands:\n runner-port-policy Print or validate a runner port policy\n with-host-lock Run a command under a host-wide lock\n host-storage Check free space on caller-supplied paths\n gha-runtime-audit Audit successful GitHub Actions job runtimes\n require-up-to-date Require HEAD to include a fetched remote branch\n fresh-base Classify the worktree against a remote base branch\n gitleaks-directory-scan Scan a directory with Gitleaks\n ast-grep-examples Run AST-grep rule examples\n ast-grep-pack Print shipped unconditional AST-grep pack paths as JSON\n gha-workspace-policy Check GitHub Actions workspace safety policy\n gha-output Write a collision-safe multiline GITHUB_OUTPUT record\n gha-needs-results Fail if required GitHub Actions job results failed\n download-with-diagnostics Download a URL and report HTTP status on failure\n download-optional-run-artifacts Download optional artifacts from the current run\n host-pressure-diagnostics Print a bounded host memory/OOM/PSI snapshot\n allocate-browser-safe-ports Allocate Fetch-safe localhost ports\n diagnose-port-collision Capture bounded localhost port diagnostics\n prepare-trivy-db Download the Trivy vulnerability database\n gha-artifacts-cleanup Delete classified GitHub Actions artifacts\n agent-issue-labels Map changed paths to labels; create missing repository labels\n http-origin Validate an optional HTTP(S) origin\n vitest-blob-manifest Stamp a vitest-blob-manifest:v1 identity file\n vitest-report-attempt Write or read a Vitest report-attempt marker\n prepare-vitest-reports Validate and select Vitest report JSON files\n pnpm-install Install a pnpm workspace with retry and release-age fail-fast\n check-cache-size Measure a path and decide whether to save a GHA cache\n make-shard-matrix Emit a [1..N] GitHub Actions shard matrix\n load-runner-env Overlay a runner env file onto GITHUB_ENV with injection guards\n clean-workspace Reset a persistent-runner workspace with a fork-PR trust gate\n install-github-release Download a checksum-verified GitHub Release binary\n run-with-timeout Run a command with GNU timeout or a Perl fallback\n lint-links Two-pass lychee: internal links fail, external warn\n materialize-pr-context Dump PR title/body/files/diff/comments and #N crawl\n wait-for-apt-locks Wait until apt/dpkg lock files are free\n install-playwright-chromium-arm64 Install Playwright Chromium from browsers.json\n ghcr-package-retention Delete old GHCR package versions (KEEP_MIN or --main-history)\n harness-admission-lane Compute a GITHUB_RUN_ID admission lane for fleet fan-out\n harness-assert-gates Fail if any named HARNESS_*_ENABLED gate is enabled\n gha-collaborator-trust Classify authors as write-access collaborators or not\n nuget-central-version Validate a Directory.Packages.props PackageVersion delta\n verify-workflow-secrets Check a secret inventory against live repository and environment secrets\n runner-policy Check workflow runner labels and jobs against a config\n dead-code Gate unused code against a committed jscpd baseline\n swift-semantic-equal Compare Swift sources ignoring comments and whitespace\n post-review Post one COMMENT review from a staged payload file\n stage-review-payload Validate a review payload file into a staging directory\n retrospective-transcript Format facts from Claude-compatible, Codex, or Grok transcripts\n link-skill Link one packaged skill into an explicit consumer directory\n retrospective-facts Gather immutable facts for a retrospective\n agent-blackboard Probe an Agent Blackboard deployment, classify distill sessions, manage snapshots\n mcp Serve the agent-blackboard journal tools over stdio (MCP)\n hook Run a repository policy module as a pre/post tool-use hook\n worktree Reclaim finished git worktrees (dry run unless --yes)\n tmux Run configured dev windows in tmux; name panes; agent reminders\n story-coverage Check component exports are imported by stories\n transient-retry Rerun a failed GitHub Actions run when an injected rule matches\n sandbox-command-audit Audit agent transcripts for sandbox escalations and denials\n cloc Count tracked lines of code by category and service with scc\n coverage-scope-check Check positive-threshold coverage rules against the Vitest scope\n trivy-policy Check tracked Trivy workflow scans and the exception registry\n docker-image-mirror Require tracked Docker images to come from a Docker Hub mirror\n workspace-registry-check Check syncpack, pnpm workspace and CI path-filter registries against tracked manifests\n final-review Post, revoke, check, or benchmark multi-pillar final reviews\n storybook-browser-runner Run the Storybook Vitest browser project with cache, retry and watchdog\n\nOptions:\n -h, --help Show this help\n -v, --version Print the package version\n\nrunner-port-policy\n (no args) Print the shipped policy as JSON\n --file <path> Validate and print a policy file\n --reserved <port> Print true if the port is reserved\n\nwith-host-lock\n --name <family>\n [--slots <n>]\n --timeout-seconds <n>\n [--command-timeout-seconds <n>]\n [--failure-diagnostics <absolute-script>]\n [--on-acquire-timeout fail|run-unlocked]\n -- <command> [args...]\n\nhost-storage --path <path> [--path <path>...] [--minimum-free-bytes <bytes>] [--pnpm-store] [--docker-root] [--docker-desktop-path <path>] [--discovery-timeout-ms <ms>]\n\ngha-runtime-audit\n [--repository owner/name] Default GITHUB_REPOSITORY\n [--branch main]\n [--median-threshold-floor <seconds>] Off unless set. Must be below the ceiling\n [--median-threshold-ceiling <seconds>] Default 360. Flags a five-sample median above this\n --pr-workflow <name|/regex/> Repeatable\n --push-workflow <name|/regex/> Repeatable\n\nrequire-up-to-date --remote <name> --branch <name>\nfresh-base [--remote <name>] [--branch <name>] [--skip-fetch]\ngitleaks-directory-scan --config <path> [--directory <path>]\nast-grep-examples --rules <directory> --config <path>\nast-grep-pack\ngha-workspace-policy [--root <directory>] [--workflow-directory <directory>] [--action-directory <directory>]\n\ngha-output <name>\ngha-needs-results [label]\ndownload-with-diagnostics <url> <destination> [-- curl-args...]\ndownload-optional-run-artifacts (--name <name>... | --pattern <pattern>) --dir <directory>\nhost-pressure-diagnostics\nallocate-browser-safe-ports <count> [--policy path] [--forbidden-ports path]\ndiagnose-port-collision [--ports \"2200 2216\"] [--output-dir PATH]\nprepare-trivy-db\ngha-artifacts-cleanup run --run-id <id> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\ngha-artifacts-cleanup sweep --older-than-hours <n> [--keep-pattern glob] [--delete-pattern glob] [--patterns-file json]\nagent-issue-labels labels-from-paths [--labeler <path>] <path>...\nagent-issue-labels ensure-labels [--repo owner/name] [--policy-file json] [--update] <label>...\nhttp-origin [--field NAME] [value]\nvitest-blob-manifest <suite> [reports-directory]\nvitest-report-attempt <write DIRECTORY SUITE|read ROOT>\nprepare-vitest-reports [primary-directory] [fallback-directory] [output-directory]\npnpm-install --runner-lifecycle persistent|ephemeral|ephemeral-full --install-scripts true|false\ncheck-cache-size <path> <max-bytes> <label>\nmake-shard-matrix <total>\nload-runner-env\nclean-workspace\ninstall-github-release --repo owner/name --version X --asset 'name-{platform}.tar.gz' --bin name [--tag-prefix PREFIX] [--expected-sha256 SHA256] [--no-checksum] [--checksums-asset NAME] [--version-flag FLAG] [--bin-dir DIR]\nrun-with-timeout <timeout-seconds> <kill-after-seconds> <command...>\nlint-links [--offline] [--config PATH] [--glob PATTERN] [files...]\nmaterialize-pr-context\nwait-for-apt-locks\ninstall-playwright-chromium-arm64 [name:archive...]\nghcr-package-retention [--main-history] [--apply|--dry-run] [<url-encoded-package>...]\nharness-admission-lane <lanes>\nharness-assert-gates <gate>...\ngha-collaborator-trust <owner/repo> [--allow-bot <login>]... < authors.json\nnuget-central-version <trusted-props> <candidate-props> <metadata-json> <output-props>\nverify-workflow-secrets [--repo OWNER/NAME] [--confirm NAME,...] [--inventory MODULE] [--export NAME] [--branch NAME]\nrunner-policy [--config FILE] [--root DIR]\ndead-code --baseline PATH [--seed|--update] [--config PATH] [--binary PATH] [--repo-root PATH] [--entry GLOB]... [--snapshot-ignore GLOB]... [--exclude GLOB]...\nswift-semantic-equal <base> <head> <file.swift>\npost-review\nstage-review-payload optional|required <source> <destination>\nretrospective-transcript [--session-id ID] [--jsonl PATH] [--projects-dir PATH] [--codex-sessions-dir PATH] [--grok-sessions-dir PATH]\nlink-skill <name> --source-root <skills-dir> --target-root <consumer-skills-dir> Link a packaged or repository-local skill\nretrospective-facts (--pr NUMBER | --branch NAME | --no-pr) [--repo OWNER/NAME] [--raw]\nagent-blackboard probe\nagent-blackboard session-start [--skip-env NAME] [--setup-hint TEXT] [--docs-path PATH] [--restore-hint TEXT] [--timeout-ms N] (SessionStart hook; payload on stdin)\nagent-blackboard distill classify <partition.jsonl> --retro-cutoff ISO8601 --session-cutoff ISO8601\nagent-blackboard snapshot partition --snapshot PATH --checksum SHA256 --counts '{\"sessions\":N,\"entries\":N,\"records\":N,\"bytes\":N}'\nagent-blackboard snapshot cleanup [--snapshot PATH] [--partition-directory PATH --receipt JSON]\nmcp\n (no args) Serve journal_append, journal_entries, outbox_status, outbox_flush,\n session_ensure, retrospective_check, retrospective_save,\n pr_validate, pr_create, pr_update,\n snapshot_export, snapshot_partition, snapshot_cleanup, and session_archive over stdio.\n Needs the optional @modelcontextprotocol/sdk peer (and zod).\n Launch it from inside the repository's git worktree.\n\nhook pre-tool-use|post-tool-use --harness claude|codex|cursor|grok --policy <module> [--root <dir>] [--automation-env NAME]...\n Reads the harness hook payload on stdin. <module> default-exports a policy object (or a function\n of the hook context returning one) with a command rule table, post-tool-use checkers, and an\n optional reminder. pre-tool-use exits 2 to block, and also when the policy cannot load.\n post-tool-use is advisory and always exits 0. --root defaults to the working directory.\n Each --automation-env NAME (repeatable) marks the run unattended when that environment variable\n is set, non-empty, and not 0 or false; CI markers and claude -p are always unattended.\n\nworktree reclaim [--yes] [--json] [--max-worktrees N] [--timeout-ms N] [--pr-limit N] [--after PATH]\n Dry run by default. Removes clean, fully pushed worktrees whose same-repository PR for the\n branch is merged or closed at exactly HEAD, using git worktree remove (never forced).\n Skips the main checkout, the current worktree, and locked, bare or detached worktrees.\n\ntransient-retry --rules <module> [--repo owner/name] [--token-env NAME] [--dry-run] <run-id>\n Loads a module exporting \"rules\" (and optionally \"limits\"), evaluates them against the failed run,\n and reruns only on a match. Exits 0 on rerun, 2 when none is warranted, 1 on error.\n\nsandbox-command-audit --rules <module> [--settings-path <file>] [--repo-root <dir>]... [--worktree-glob <glob>]... [--projects-dir <dir>] [--codex-sessions-dir <dir>] [--session-id <id>] [--limit N] [--since DAYS] [--json] [--raw]\n Scans Claude and Codex transcripts scoped to this repository (git roots plus any --worktree-glob\n layouts) and reports Bash escalations, denials and permission failures by command prefix.\n <module> exports an object (default or \"rules\") with loadPolicy({ settingsPath, cwd }), which\n returns { covers(list, commandOrPrefix) } for the allow/deny/excluded lists or { error }, and\n optionally classifyFailure(command, text, source), limits and worktreeGlobs. Transcript roots\n default to CLAUDE_CONFIG_DIR/projects and CODEX_HOME/sessions. --raw prints full command text.\n\ncloc --config <file> [--scc-bin <path>] [--json]\n Counts git-tracked lines with scc (SCC_BIN or --scc-bin, else \"scc\"), classified into\n source/tests/tooling and by service, and prints an aligned table (--json: rows). <file> is a .json\n file or a module whose default (or \"config\") export is the layout: services [{ name, match }],\n defaultService, categories, testPatterns, toolingPatterns, countedRoots, countedRootFiles,\n excludedDirs, ignoredDirs, sccBin, missingSccHint.\n\nfinal-review post --repo owner/name --pr N --head-sha SHA --input findings.json [--dry-run]\nfinal-review diff --repo owner/name --pr N\nfinal-review revoke --repo owner/name --pr N\nfinal-review status --repo owner/name --pr N\nfinal-review bench --cases cases.json --prompts DIR --out DIR [--pillar NAME]... [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL]\nfinal-review bench-verifier --cases cases.json --findings-from DIR --prompts DIR --out DIR [--case ID]... [--model NAME] [--jobs N] [--max-turns N] [--score-only] [--judge MODEL] [--checker MODEL]\n post renders verified findings as one or more COMMENT reviews (verdict, reviewed SHA and effort in\n the header; inline comments where the diff allows, the rest in the body) and adds the \"final\n approval\" label on Approve or removes it otherwise. It refuses when the head moved. diff prints the pull request\n diff with base and head line numbers on every line, for reviewers to cite. revoke removes\n the label before a re-review. status exits 0 only when the newest final review approved the current\n head and the label is present. bench runs each pillar prompt against labeled cases from a git-free\n snapshot of the pre-merge head and scores recall and precision; --judge grades by claim\n with a model instead of by location. bench-verifier runs only the verifier, one call per case\n and no subagents, on the pillar findings a bench --out DIR saved (each judged-real finding is\n also planted under a second pillar), and scores real findings kept, label recall, severity\n agreement, duplicates merged, and other blocking and high findings confirmed by --checker;\n --prompts is the skill's references directory.\n\ncoverage-scope-check [--rules <path>] [--scope-file <json>] [--include <glob>]... [--exclude <glob>]...\n Fails (exit 1) when a git-tracked file that a coverage rules file (default .coverage-rules.yml)\n gives a patch_coverage_min above 0, and its scope does not ignore, is outside the collector's\n scope. The collector scope is the repeated --include/--exclude globs plus, optionally, the\n include/exclude string arrays of a --scope-file JSON written from the Vitest coverage config.\n At least one include glob is required.\n\ntrivy-policy [--root <dir>] [--ignore-file <path>] [--findings-exit-code-env <NAME>] [--cleanup-url-prefix <url>] [--min-gates N] [--min-sboms N] [--max-expiry-days N]\n Reads git-tracked workflows, composite actions and shell scripts and prints each violation (exit 1).\n Every Trivy package scan must be an auditable CLI run block (no action, wrapper, substitution or\n background). Vulnerability gates need --scanners vuln, --pkg-types os, --severity CRITICAL,HIGH,\n --ignore-unfixed, --skip-db-update, --quiet, table output to a file, --ignorefile <ignore-file>\n (default .trivyignore.yaml) and --exit-code \"$<NAME>\" (default TRIVY_FINDINGS_EXIT_CODE).\n CycloneDX SBOM scans must be unfiltered with --exit-code 0. At least --min-gates gates and\n --min-sboms SBOM scans (default 2 each) must exist. The tracked ignore registry needs purl-exact OS\n packages, a \"Not exploitable: ... Cleanup: <issue URL>\" statement (URL must start with\n --cleanup-url-prefix when given) and an expired_at between tomorrow and --max-expiry-days (30).\n\ndocker-image-mirror [--root <dir>] [--mirror <host>] [--allow-registry <prefix>]... [--require-digest]\n [--allow-ref-expression <text>]...\n Prints each Docker Hub image in git-tracked workflows, composite actions and Dockerfiles (exit 1):\n run, pull and create commands, containers, services, docker:// references, buildx drivers, FROM,\n COPY --from and # syntax. Images must name a registry host; --mirror (default mirror.gcr.io) is\n the host the message tells you to use. --allow-registry (repeatable) instead requires every image,\n including FROM and COPY --from, to start with one of the prefixes; a reference that expands a\n variable passes only when its literal text before the variable starts with a prefix. --require-digest\n requires @sha256: plus 64 lowercase hex characters. Stage aliases and scratch stay exempt.\n --allow-ref-expression (repeatable) is an audited escape hatch for dynamic job outputs: a reference\n whose raw text exactly equals the value is accepted. Images the same file builds locally are exempt.\n\nworkspace-registry-check [--root <dir>] [--syncpack-config <path>] [--workspace-file <path>]\n [--path-filters <path>] [--tooling-group <name>] [--trigger-only-glob <glob>] [--runtime-prefix <prefix>]\n Reconciles git-tracked package.json manifests with the syncpack \"source\" globs (default\n .syncpackrc.json), the pnpm \"packages\" globs (pnpm-workspace.yaml) and the dorny/paths-filter groups\n (.github/ci-path-filters.yml) and prints one line per violation (exit 1). The --tooling-group group\n (default tooling) must reach every manifest and wildcard workspace glob, and its --trigger-only-glob\n (default **/package.json) does not count as registering a manifest. Each group named with\n --runtime-prefix (default runtime-) must mirror the primary group of the same name.\n\nstorybook-browser-runner --project <name> [--command <cmd>] [--result-file <path>]... [--completion-file <path>] [--coverage-arg=<arg>]... [--optimize-deps-config <path>] [--env-prefix <PREFIX>] [--progress-prefix <text>] [-- <argv>...]\n Spawns <cmd> (default pnpm) in its own process group with <argv> (default \"exec vitest run\n --project <name>\") plus the coverage args (default --coverage) and supervises it: a Vite optimizer\n cache, retries on known Vite and browser start-up flakes, and stall, hang and session-budget\n watchdogs. --result-file paths are removed before each attempt and --completion-file is the one\n whose appearance is recorded. Tuned by <PREFIX>_STARTUP_ATTEMPTS, _OPTIMIZER_STALL_MS, _HANG_MS,\n _SESSION_BUDGET_MS, _DIAGNOSTICS_FILE, _PERSISTENT_CACHE, _PERSISTENT_CACHE_DIR and _COVERAGE\n (PREFIX defaults to STORYBOOK_BROWSER). Exits with the resolved test exit code.\n\ntmux up --config <file> [--root <dir>] [--no-attach] [-v|--verbose]\ntmux name [--socket PATH --pane %ID --worktree PATH] <name> | read-title | post-tool | agent-reminder <event>\n Environment: TMUX_TARGET_BIN, TMUX_TARGET_ROOT, TMUX_NAME_COMMAND, AGENT_TMUX_SOCKET/PANE/WORKTREE.\n\nstory-coverage --root <dir> --component-root <rel> --story-root <rel> --source-extension <ext> --component-extension <ext> --marker <regex> --story-pattern <regex> [--alias <prefix>=<path>] [--exclusions <json>] [--exclusions-label <text>]\n";
|
|
@@ -0,0 +1,23 @@
|
|
|
1
|
+
import { type Pillar, type Severity } from '../finding.mts';
|
|
2
|
+
export type ExpectedFinding = {
|
|
3
|
+
pillar: Pillar;
|
|
4
|
+
severity: Severity;
|
|
5
|
+
path: string | null;
|
|
6
|
+
lines: [number, number] | null;
|
|
7
|
+
claim: string;
|
|
8
|
+
};
|
|
9
|
+
export type EvalCase = {
|
|
10
|
+
id: string;
|
|
11
|
+
repo: string;
|
|
12
|
+
pr: number;
|
|
13
|
+
title: string;
|
|
14
|
+
baseSha: string;
|
|
15
|
+
headSha: string;
|
|
16
|
+
kind: 'positive' | 'negative';
|
|
17
|
+
reason: string;
|
|
18
|
+
expected: ExpectedFinding[];
|
|
19
|
+
};
|
|
20
|
+
/** Parses labeled eval cases; ids must be unique. */
|
|
21
|
+
export declare function parseEvalCases(raw: unknown): EvalCase[];
|
|
22
|
+
/** Cases that exercise one pillar: its positives plus every negative. */
|
|
23
|
+
export declare function casesForPillar(cases: readonly EvalCase[], pillar: Pillar): EvalCase[];
|
|
@@ -0,0 +1,89 @@
|
|
|
1
|
+
import { FinalReviewError, PILLARS, SEVERITIES } from "../finding.mjs";
|
|
2
|
+
const SHA_RE = /^[0-9a-f]{40}$/u;
|
|
3
|
+
const REPO_RE = /^[A-Za-z0-9_.-]+\/[A-Za-z0-9_.-]+$/u;
|
|
4
|
+
function isRecord(value) {
|
|
5
|
+
return value !== null && typeof value === 'object' && !Array.isArray(value);
|
|
6
|
+
}
|
|
7
|
+
function fail(where, message) {
|
|
8
|
+
throw new FinalReviewError(`${where}: ${message}`);
|
|
9
|
+
}
|
|
10
|
+
function str(record, field, where) {
|
|
11
|
+
const value = record[field];
|
|
12
|
+
if (typeof value !== 'string')
|
|
13
|
+
fail(where, `${field} must be a string`);
|
|
14
|
+
return value;
|
|
15
|
+
}
|
|
16
|
+
function lines(value, where) {
|
|
17
|
+
if (value === null || value === undefined)
|
|
18
|
+
return null;
|
|
19
|
+
if (!Array.isArray(value) || value.length !== 2)
|
|
20
|
+
fail(where, 'lines must be [start, end] or null');
|
|
21
|
+
const [start, end] = value;
|
|
22
|
+
if (!Number.isInteger(start) ||
|
|
23
|
+
!Number.isInteger(end) ||
|
|
24
|
+
start < 1 ||
|
|
25
|
+
end < start) {
|
|
26
|
+
fail(where, 'lines must be ascending positive integers');
|
|
27
|
+
}
|
|
28
|
+
return [start, end];
|
|
29
|
+
}
|
|
30
|
+
function expected(raw, where) {
|
|
31
|
+
if (!isRecord(raw))
|
|
32
|
+
fail(where, 'must be an object');
|
|
33
|
+
const pillar = PILLARS.find((name) => name === raw.pillar) ?? fail(where, 'unknown pillar');
|
|
34
|
+
const severity = SEVERITIES.find((name) => name === raw.severity) ?? fail(where, 'unknown severity');
|
|
35
|
+
const path = typeof raw.path === 'string' && raw.path.length > 0 ? raw.path : null;
|
|
36
|
+
return { pillar, severity, path, lines: lines(raw.lines, where), claim: str(raw, 'claim', where) };
|
|
37
|
+
}
|
|
38
|
+
function evalCase(raw, index) {
|
|
39
|
+
const where = `cases[${index}]`;
|
|
40
|
+
if (!isRecord(raw))
|
|
41
|
+
fail(where, 'must be an object');
|
|
42
|
+
const repo = str(raw, 'repo', where);
|
|
43
|
+
if (!REPO_RE.test(repo))
|
|
44
|
+
fail(where, 'repo must be owner/name');
|
|
45
|
+
const baseSha = str(raw, 'baseSha', where);
|
|
46
|
+
const headSha = str(raw, 'headSha', where);
|
|
47
|
+
if (!SHA_RE.test(baseSha) || !SHA_RE.test(headSha))
|
|
48
|
+
fail(where, 'baseSha and headSha must be full SHAs');
|
|
49
|
+
if (raw.kind !== 'positive' && raw.kind !== 'negative')
|
|
50
|
+
fail(where, 'kind must be positive or negative');
|
|
51
|
+
if (!Number.isInteger(raw.pr) || raw.pr <= 0)
|
|
52
|
+
fail(where, 'pr must be a positive integer');
|
|
53
|
+
const items = Array.isArray(raw.expected)
|
|
54
|
+
? raw.expected
|
|
55
|
+
: fail(where, 'expected must be an array');
|
|
56
|
+
const parsed = items.map((item, at) => expected(item, `${where}.expected[${at}]`));
|
|
57
|
+
if (raw.kind === 'positive' && parsed.length === 0)
|
|
58
|
+
fail(where, 'a positive case needs an expected finding');
|
|
59
|
+
if (raw.kind === 'negative' && parsed.length > 0)
|
|
60
|
+
fail(where, 'a negative case has no expected findings');
|
|
61
|
+
return {
|
|
62
|
+
id: str(raw, 'id', where),
|
|
63
|
+
repo,
|
|
64
|
+
pr: raw.pr,
|
|
65
|
+
title: str(raw, 'title', where),
|
|
66
|
+
baseSha,
|
|
67
|
+
headSha,
|
|
68
|
+
kind: raw.kind,
|
|
69
|
+
reason: typeof raw.reason === 'string' ? raw.reason : '',
|
|
70
|
+
expected: parsed,
|
|
71
|
+
};
|
|
72
|
+
}
|
|
73
|
+
/** Parses labeled eval cases; ids must be unique. */
|
|
74
|
+
export function parseEvalCases(raw) {
|
|
75
|
+
if (!Array.isArray(raw))
|
|
76
|
+
throw new FinalReviewError('Eval cases must be a JSON array.');
|
|
77
|
+
const cases = raw.map(evalCase);
|
|
78
|
+
const seen = new Set();
|
|
79
|
+
for (const item of cases) {
|
|
80
|
+
if (seen.has(item.id))
|
|
81
|
+
throw new FinalReviewError(`Duplicate eval case id: ${item.id}`);
|
|
82
|
+
seen.add(item.id);
|
|
83
|
+
}
|
|
84
|
+
return cases;
|
|
85
|
+
}
|
|
86
|
+
/** Cases that exercise one pillar: its positives plus every negative. */
|
|
87
|
+
export function casesForPillar(cases, pillar) {
|
|
88
|
+
return cases.filter((item) => item.kind === 'negative' || item.expected.some((entry) => entry.pillar === pillar));
|
|
89
|
+
}
|
|
@@ -0,0 +1,12 @@
|
|
|
1
|
+
import { prepareSnapshot, type Exec } from './snapshot.mts';
|
|
2
|
+
export type BenchDeps = {
|
|
3
|
+
exec: Exec;
|
|
4
|
+
readFile(path: string): string;
|
|
5
|
+
writeFile(path: string, text: string): void;
|
|
6
|
+
exists(path: string): boolean;
|
|
7
|
+
log(line: string): void;
|
|
8
|
+
prepare: typeof prepareSnapshot;
|
|
9
|
+
};
|
|
10
|
+
export declare const defaultBenchDeps: BenchDeps;
|
|
11
|
+
/** `vouchington final-review bench`: run pillar prompts over labeled cases and score them. */
|
|
12
|
+
export declare function runFinalReviewBenchCli(args: string[], deps?: BenchDeps): Promise<number>;
|
|
@@ -0,0 +1,144 @@
|
|
|
1
|
+
import { existsSync, mkdirSync, readFileSync, writeFileSync } from 'node:fs';
|
|
2
|
+
import { dirname, join } from 'node:path';
|
|
3
|
+
import { parseArgs } from 'node:util';
|
|
4
|
+
import { PILLARS } from "../finding.mjs";
|
|
5
|
+
import { casesForPillar, parseEvalCases } from "./cases.mjs";
|
|
6
|
+
import { currentAssignment, judgeAll, pool } from "./judge-all.mjs";
|
|
7
|
+
import { runPillar } from "./runner.mjs";
|
|
8
|
+
import { renderSummaryTable, scoreCase, summarizePillar } from "./score.mjs";
|
|
9
|
+
import { defaultExec, prepareSnapshot } from "./snapshot.mjs";
|
|
10
|
+
export const defaultBenchDeps = {
|
|
11
|
+
exec: defaultExec,
|
|
12
|
+
readFile: (path) => readFileSync(path, 'utf8'),
|
|
13
|
+
writeFile: (path, text) => {
|
|
14
|
+
mkdirSync(dirname(path), { recursive: true });
|
|
15
|
+
writeFileSync(path, text);
|
|
16
|
+
},
|
|
17
|
+
exists: existsSync,
|
|
18
|
+
log: (line) => process.stderr.write(`${line}\n`),
|
|
19
|
+
prepare: prepareSnapshot,
|
|
20
|
+
};
|
|
21
|
+
function pickPillars(requested, cases) {
|
|
22
|
+
if (requested.length > 0) {
|
|
23
|
+
return requested.map((name) => {
|
|
24
|
+
const pillar = PILLARS.find((candidate) => candidate === name);
|
|
25
|
+
if (pillar === undefined)
|
|
26
|
+
throw new Error(`Unknown pillar: ${name}`);
|
|
27
|
+
return pillar;
|
|
28
|
+
});
|
|
29
|
+
}
|
|
30
|
+
const used = new Set(cases.flatMap((item) => item.expected.map((entry) => entry.pillar)));
|
|
31
|
+
return PILLARS.filter((pillar) => used.has(pillar));
|
|
32
|
+
}
|
|
33
|
+
async function runAll(cases, pillars, options, deps) {
|
|
34
|
+
const rawPath = (pillar, id) => join(options.out, 'raw', pillar, `${id}.json`);
|
|
35
|
+
if (options.scoreOnly)
|
|
36
|
+
return;
|
|
37
|
+
const snapshots = new Map();
|
|
38
|
+
for (const item of new Set(pillars.flatMap((pillar) => casesForPillar(cases, pillar)))) {
|
|
39
|
+
deps.log(`snapshot ${item.id}`);
|
|
40
|
+
snapshots.set(item.id, (await deps.prepare(item, join(options.out, 'cache'), join(options.out, 'work'), deps.exec))
|
|
41
|
+
.dir);
|
|
42
|
+
}
|
|
43
|
+
const contractPath = join(options.prompts, '_contract.md');
|
|
44
|
+
const contract = deps.exists(contractPath) ? deps.readFile(contractPath).trim() : '';
|
|
45
|
+
const tasks = pillars.flatMap((pillar) => {
|
|
46
|
+
const pillarPrompt = [contract, deps.readFile(join(options.prompts, `${pillar}.md`))]
|
|
47
|
+
.filter((part) => part.length > 0)
|
|
48
|
+
.join('\n\n');
|
|
49
|
+
return casesForPillar(cases, pillar).map((item) => ({ pillar, pillarPrompt, item }));
|
|
50
|
+
});
|
|
51
|
+
await pool(tasks, options.jobs, async ({ pillar, pillarPrompt, item }) => {
|
|
52
|
+
deps.log(`run ${pillar} ${item.id}`);
|
|
53
|
+
const dir = snapshots.get(item.id);
|
|
54
|
+
let result;
|
|
55
|
+
try {
|
|
56
|
+
const run = { pillar, pillarPrompt, model: options.model, maxTurns: options.maxTurns };
|
|
57
|
+
result = { findings: await runPillar(item, dir, run, deps.exec) };
|
|
58
|
+
}
|
|
59
|
+
catch (error) {
|
|
60
|
+
result = { error: error instanceof Error ? error.message : String(error) };
|
|
61
|
+
}
|
|
62
|
+
deps.writeFile(rawPath(pillar, item.id), `${JSON.stringify(result, null, 2)}\n`);
|
|
63
|
+
});
|
|
64
|
+
}
|
|
65
|
+
function scoreAll(cases, pillars, options, deps) {
|
|
66
|
+
const { out, judge } = options;
|
|
67
|
+
const report = pillars.map((pillar) => {
|
|
68
|
+
const scores = [];
|
|
69
|
+
const errors = [];
|
|
70
|
+
for (const item of casesForPillar(cases, pillar)) {
|
|
71
|
+
const path = join(out, 'raw', pillar, `${item.id}.json`);
|
|
72
|
+
if (!deps.exists(path)) {
|
|
73
|
+
errors.push(`${item.id}: not run`);
|
|
74
|
+
continue;
|
|
75
|
+
}
|
|
76
|
+
const raw = JSON.parse(deps.readFile(path));
|
|
77
|
+
if ('error' in raw)
|
|
78
|
+
errors.push(`${item.id}: ${raw.error}`);
|
|
79
|
+
else if (judge === undefined)
|
|
80
|
+
scores.push(scoreCase(item, pillar, raw.findings));
|
|
81
|
+
else {
|
|
82
|
+
const expected = item.expected.filter((entry) => entry.pillar === pillar);
|
|
83
|
+
const assignment = currentAssignment(raw, expected, judge);
|
|
84
|
+
// Unjudged cases stay out of judged metrics rather than falling back to locations.
|
|
85
|
+
if (assignment === undefined)
|
|
86
|
+
errors.push(`${item.id}: not judged`);
|
|
87
|
+
else
|
|
88
|
+
scores.push(scoreCase(item, pillar, raw.findings, assignment));
|
|
89
|
+
}
|
|
90
|
+
}
|
|
91
|
+
return { summary: summarizePillar(pillar, scores), scores, errors };
|
|
92
|
+
});
|
|
93
|
+
deps.writeFile(join(out, 'summary.json'), `${JSON.stringify(report, null, 2)}\n`);
|
|
94
|
+
const errorLines = report.flatMap((entry) => entry.errors.map((line) => `- ${entry.summary.pillar} ${line}`));
|
|
95
|
+
const matching = judge === undefined ? 'file and line' : `claims judged by ${judge}`;
|
|
96
|
+
const markdown = [
|
|
97
|
+
`Matching: ${matching}.`,
|
|
98
|
+
renderSummaryTable(report.map((entry) => entry.summary)),
|
|
99
|
+
...(errorLines.length > 0 ? ['Errors and skipped runs:', errorLines.join('\n')] : []),
|
|
100
|
+
].join('\n\n');
|
|
101
|
+
deps.writeFile(join(out, 'summary.md'), `${markdown}\n`);
|
|
102
|
+
return markdown;
|
|
103
|
+
}
|
|
104
|
+
/** `vouchington final-review bench`: run pillar prompts over labeled cases and score them. */
|
|
105
|
+
export async function runFinalReviewBenchCli(args, deps = defaultBenchDeps) {
|
|
106
|
+
const { values } = parseArgs({
|
|
107
|
+
args,
|
|
108
|
+
strict: true,
|
|
109
|
+
options: {
|
|
110
|
+
cases: { type: 'string' },
|
|
111
|
+
prompts: { type: 'string' },
|
|
112
|
+
out: { type: 'string' },
|
|
113
|
+
pillar: { type: 'string', multiple: true },
|
|
114
|
+
case: { type: 'string', multiple: true },
|
|
115
|
+
model: { type: 'string' },
|
|
116
|
+
'max-turns': { type: 'string' },
|
|
117
|
+
jobs: { type: 'string' },
|
|
118
|
+
'score-only': { type: 'boolean' },
|
|
119
|
+
judge: { type: 'string' },
|
|
120
|
+
},
|
|
121
|
+
});
|
|
122
|
+
if (values.cases === undefined || values.out === undefined)
|
|
123
|
+
throw new Error('bench needs --cases and --out.');
|
|
124
|
+
if (values['score-only'] !== true && values.prompts === undefined)
|
|
125
|
+
throw new Error('bench needs --prompts.');
|
|
126
|
+
const ids = new Set(values.case ?? []);
|
|
127
|
+
const cases = parseEvalCases(JSON.parse(deps.readFile(values.cases))).filter((item) => ids.size === 0 || ids.has(item.id));
|
|
128
|
+
const pillars = pickPillars(values.pillar ?? [], cases);
|
|
129
|
+
const options = {
|
|
130
|
+
out: values.out,
|
|
131
|
+
prompts: values.prompts ?? '',
|
|
132
|
+
model: values.model ?? 'sonnet',
|
|
133
|
+
maxTurns: Number(values['max-turns'] ?? '40'),
|
|
134
|
+
jobs: Number(values.jobs ?? '3'),
|
|
135
|
+
scoreOnly: values['score-only'] === true,
|
|
136
|
+
judge: values.judge,
|
|
137
|
+
};
|
|
138
|
+
await runAll(cases, pillars, options, deps);
|
|
139
|
+
if (options.judge !== undefined) {
|
|
140
|
+
await judgeAll(cases, pillars, { out: options.out, model: options.judge, jobs: options.jobs }, deps);
|
|
141
|
+
}
|
|
142
|
+
process.stdout.write(`${scoreAll(cases, pillars, options, deps)}\n`);
|
|
143
|
+
return 0;
|
|
144
|
+
}
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
import type { Finding, Pillar } from '../finding.mts';
|
|
2
|
+
import { type EvalCase, type ExpectedFinding } from './cases.mts';
|
|
3
|
+
import { type Assignment, type Judged } from './judge.mts';
|
|
4
|
+
import type { Exec } from './snapshot.mts';
|
|
5
|
+
export type RawResult = {
|
|
6
|
+
findings: Finding[];
|
|
7
|
+
judged?: Judged;
|
|
8
|
+
} | {
|
|
9
|
+
error: string;
|
|
10
|
+
};
|
|
11
|
+
export type JudgeDeps = {
|
|
12
|
+
exec: Exec;
|
|
13
|
+
readFile(path: string): string;
|
|
14
|
+
writeFile(path: string, text: string): void;
|
|
15
|
+
exists(path: string): boolean;
|
|
16
|
+
log(line: string): void;
|
|
17
|
+
};
|
|
18
|
+
/** Fingerprints what a judgement indexes, so edited claims or re-run findings invalidate it. */
|
|
19
|
+
export declare function judgeInputs(expected: readonly ExpectedFinding[], findings: readonly Finding[]): string;
|
|
20
|
+
/** The cached assignment when it was made by `model` for exactly these inputs, else undefined. */
|
|
21
|
+
export declare function currentAssignment(raw: {
|
|
22
|
+
findings: Finding[];
|
|
23
|
+
judged?: Judged;
|
|
24
|
+
}, expected: readonly ExpectedFinding[], model: string): Assignment | undefined;
|
|
25
|
+
/** Runs `work` over `items` with at most `jobs` in flight. */
|
|
26
|
+
export declare function pool<T>(items: readonly T[], jobs: number, work: (item: T) => Promise<void>): Promise<void>;
|
|
27
|
+
/** Adds a judge assignment to every raw result that lacks a current one from this model. */
|
|
28
|
+
export declare function judgeAll(cases: readonly EvalCase[], pillars: readonly Pillar[], options: {
|
|
29
|
+
out: string;
|
|
30
|
+
model: string;
|
|
31
|
+
jobs: number;
|
|
32
|
+
}, deps: JudgeDeps): Promise<void>;
|