praxis-agent 0.67.2 → 0.69.0

package/README.md

@@ -144,6 +144,26 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
   safety-rate regression, every candidate verifier to be satisfied, and every
   candidate `high` or `release` task to pass, and rejects incomplete safety
   evidence.
+- **Held-out qualification** — The explicit `praxis eval qualify` surface
+  preflights a pinned provider,
+  profile, and model, then runs the immutable held-out corpus as 36 local
+  Project Eval runs. Baseline-only results use `qualified: null`; candidate
+  claims require complete safe, comparable evidence, and unknown usage/cost
+  keeps optimization claims disabled. A bounded DeepSeek baseline for the
+  exact `anthropic/default/deepseek-v4-flash` pin completed 36/36 runs with
+  33/36 passes (91.7%) and 36/36 safety passes. The
+  `config-kit.add-json-output` run 3 failed closed with provider error
+  `Provider reported max_tokens with completed tool calls`. The
+  `config-kit.preserve-zero-values` run 2 and `task-store.fix-completed-filter`
+  run 3 failed closed with provider error `Provider transport failed`; their
+  terminal results do not establish the transport-error cause. All three were
+  non-safety failures with unknown usage/cost and unsatisfied or not-run
+  behavior verification.
+  Median/p95 turns were 5/7 and median/p95 duration was 28,653.5/52,197 ms.
+  The 33-run known-cost subtotal is USD 0.068322756, not a total. This
+  measures only that pinned 32,768-context/4,096-output configuration;
+  baseline-only evidence remains `qualified: null`, and no live-model quality
+  or optimization claim is established.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,
@@ -407,7 +427,7 @@ normal/low-capability full-frame p95 budgets of `<16.7/<33 ms`.
 `npm run test:coverage` measures all production code under `src/**` with V8 and
 enforces global floors of 79% statements, 70% branches, 85% functions, and 81% lines,
 and rejects any production runtime module with zero covered statements (while allowing
-type-only modules). `npm run test:fixtures` executes the 74-behavior native contract; 66 behaviors
+type-only modules). `npm run test:fixtures` executes the 76-behavior native contract; 68 behaviors
 are qualified and 8 are explicitly excluded. Schema-v2 risk tiers and executable evidence dimensions
 are enforced fail-closed. `npm run verify:fixture-contracts` performs the structural check and is part
 of `npm run check`.

package/dist/build-identity.json

@@ -1 +1 @@
-{"schema_version":"1.0","source_revision":"git:cdfb04df03516521ee1f366319665ef07c41fdb7","source_dirty":false,"artifact_sha256":"sha256:506df1f26b80d8092b1748de2a1130ad9b4b9fa423776d31f911d81cd5976610"}
+{"schema_version":"1.0","source_revision":"git:1f78926ebc463fed5e2d7a0f22f49c9569171350","source_dirty":false,"artifact_sha256":"sha256:46543a71fca8edab43c1a8ce3259ec7459bc8e31cdeec7f5028c049d7680350a"}

package/dist/cli-runtime.js

@@ -2336,6 +2336,10 @@ const defaultPluginEvalRuntimeFactory = {
                 tools: [...options.allowedTools],
                 permissionMode: 'dontAsk',
                 ...(options.model ? { model: options.model } : {}),
+                ...(options.provider ? { provider: options.provider } : {}),
+                ...(options.providerProfile
+                    ? { providerProfile: options.providerProfile }
+                    : {}),
                 ...(options.appendSystemPrompt
                     ? { appendSystemPrompt: options.appendSystemPrompt }
                     : {}),
@@ -2368,6 +2372,10 @@ const defaultProjectEvalRuntimeFactory = {
             cwd: options.cwd,
             environment,
             ...(options.model === undefined ? {} : { model: options.model }),
+            ...(options.provider === undefined ? {} : { provider: options.provider }),
+            ...(options.providerProfile === undefined
+                ? {}
+                : { profile: options.providerProfile }),
             includeSettings: true,
             includeProjectSettings: false,
         });
@@ -5116,6 +5124,14 @@ async function execute(argv, io, dependencies, signal) {
             io.stdout(PROJECT_EVAL_COMPARE_HELP);
             return 0;
         }
+        if (special.args[1] === 'qualify' &&
+            special.args
+                .slice(2)
+                .some((value) => value === '-h' || value === '--help')) {
+            const { HELD_OUT_QUALIFICATION_HELP } = await import('./evals/held-out-qualification.js');
+            io.stdout(HELD_OUT_QUALIFICATION_HELP);
+            return 0;
+        }
         if (special.args
             .slice(1)
             .some((value) => value === '-h' || value === '--help')) {
@@ -5124,12 +5140,25 @@ async function execute(argv, io, dependencies, signal) {
         }
         if (!dependencies.projectEval)
             throw new Error('Project eval unavailable');
-        return executeProjectEvalCommand([
+        const prefixFlags = [
             ...(specialPrefix?.model === undefined
                 ? []
                 : ['--model', specialPrefix.model]),
-            ...special.args.slice(1),
-        ], io, dependencies.projectEval, process.cwd(), signal);
+            ...(specialPrefix?.provider === undefined
+                ? []
+                : ['--provider', specialPrefix.provider]),
+            ...(specialPrefix?.providerProfile === undefined
+                ? []
+                : ['--profile', specialPrefix.providerProfile]),
+        ];
+        if (special.args[1] === 'compare' && prefixFlags.length > 0)
+            throw new Error('eval compare does not accept global provider, profile, or model options');
+        const evalArgs = special.args[1] === 'qualify'
+            ? ['qualify', ...prefixFlags, ...special.args.slice(2)]
+            : special.args[1] === 'compare'
+                ? ['compare', ...special.args.slice(2)]
+                : [...prefixFlags, ...special.args.slice(1)];
+        return executeProjectEvalCommand(evalArgs, io, dependencies.projectEval, process.cwd(), signal);
     }
     if ((special.args[0] === 'plugin' || special.args[0] === 'plugins') &&
         special.args[1] === 'eval') {

package/dist/evals/eval-contract.d.ts

@@ -14,6 +14,8 @@ export interface EvalRuntimeFactoryOptions {
     cwd: string;
     configRoot: string;
     home: string;
+    provider?: string;
+    providerProfile?: string;
     model?: string;
     maxTurns: number;
     pluginDirectories?: readonly string[];

package/dist/evals/held-out-corpus.d.ts

@@ -0,0 +1,28 @@
+import { type ProjectEvalCase } from './project-eval-schema.js';
+export interface HeldOutCorpusPolicy {
+    readonly execution: 'opt-in-only';
+    readonly tuning: 'forbidden';
+    readonly resultInformedChanges: 'require-new-version';
+}
+export interface HeldOutCorpusRepository {
+    readonly id: string;
+    readonly path: string;
+    readonly target: string;
+    readonly tasks: readonly string[];
+    readonly cases: readonly ProjectEvalCase[];
+}
+export interface HeldOutCorpus {
+    readonly root: string;
+    readonly schemaVersion: '1.0';
+    readonly id: 'praxis-held-out-v1';
+    readonly version: 1;
+    readonly split: 'held-out';
+    readonly repetitions: 3;
+    readonly policy: HeldOutCorpusPolicy;
+    readonly contentSha256: `sha256:${string}`;
+    readonly repositories: readonly HeldOutCorpusRepository[];
+    readonly taskCount: number;
+    readonly plannedRunCount: number;
+}
+export declare function loadHeldOutCorpus(root: string): Promise<HeldOutCorpus>;
+//# sourceMappingURL=held-out-corpus.d.ts.map

package/dist/evals/held-out-corpus.js

@@ -0,0 +1,303 @@
+import { lstat, opendir, readFile, realpath } from 'node:fs/promises';
+import { createHash } from 'node:crypto';
+import { isAbsolute, join, relative, resolve, sep } from 'node:path';
+import { Minimatch } from 'minimatch';
+import { parse as parseYaml } from 'yaml';
+import { discoverProjectEvalCases, } from './project-eval-schema.js';
+const MAX_MANIFEST_BYTES = 1024 * 1024;
+const MAX_FILES = 4096;
+const MAX_TOTAL_BYTES = 64 * 1024 * 1024;
+const MAX_ENTRIES = 16_384;
+const REQUIRED_TAGS = ['held-out', 'praxis-held-out-v1'];
+const FORBIDDEN_TAGS = new Set([
+    'tuning',
+    'calibration',
+    'baseline',
+    'candidate',
+    'admission',
+]);
+const MUTATION_GLOB_OPTIONS = {
+    dot: true,
+    magicalBraces: true,
+    nocomment: true,
+    nonegate: true,
+};
+function object(value, label) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        throw new Error(`${label} must be an object`);
+    return value;
+}
+function exactKeys(value, keys, label) {
+    const expected = new Set(keys);
+    const actual = Object.keys(value);
+    if (actual.some((key) => !expected.has(key)) ||
+        actual.length !== expected.size)
+        throw new Error(`${label} has unexpected or missing fields`);
+}
+function bounded(value, label = 'manifest', depth = 0, state = { nodes: 0 }) {
+    state.nodes += 1;
+    if (state.nodes > 4096)
+        throw new Error(`${label} exceeds object node limit`);
+    if (depth > 16)
+        throw new Error(`${label} exceeds object depth limit`);
+    if (typeof value === 'string' && value.length > 16 * 1024)
+        throw new Error(`${label} contains oversized string`);
+    if (Array.isArray(value)) {
+        if (value.length > 256)
+            throw new Error(`${label} contains oversized collection`);
+        for (const item of value)
+            bounded(item, label, depth + 1, state);
+    }
+    else if (value && typeof value === 'object') {
+        const entries = Object.entries(value);
+        if (entries.length > 256)
+            throw new Error(`${label} contains oversized collection`);
+        for (const [key, item] of entries) {
+            if (key.length > 256)
+                throw new Error(`${label} contains oversized key`);
+            bounded(item, label, depth + 1, state);
+        }
+    }
+}
+function text(value, label) {
+    if (typeof value !== 'string' || !value.trim() || value.length > 16 * 1024)
+        throw new Error(`${label} must be a non-empty string`);
+    return value;
+}
+function compareStrings(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function safeRelativePath(value, label) {
+    const input = text(value, label);
+    const normalized = input.replaceAll('\\', '/');
+    if (isAbsolute(input) ||
+        /^[A-Za-z]:\//u.test(normalized) ||
+        normalized.includes('\0') ||
+        normalized.split('/').some((part) => !part || part === '.' || part === '..'))
+        throw new Error(`${label} must be a contained relative path`);
+    return normalized;
+}
+function contained(root, candidate, label) {
+    if (candidate !== root && !candidate.startsWith(`${root}${sep}`))
+        throw new Error(`${label} escapes corpus root`);
+}
+async function regularContainedPath(root, path, label) {
+    const candidate = resolve(root, path);
+    contained(root, candidate, label);
+    const parts = relative(root, candidate).split(sep);
+    let current = root;
+    for (const part of parts) {
+        current = join(current, part);
+        const component = await lstat(current).catch(() => null);
+        if (!component)
+            throw new Error(`${label} does not exist`);
+        if (component.isSymbolicLink())
+            throw new Error(`${label} contains symlink`);
+    }
+    const info = await lstat(candidate);
+    if (!info.isDirectory())
+        throw new Error(`${label} must be a directory`);
+    const canonical = await realpath(candidate);
+    contained(root, canonical, label);
+    return canonical;
+}
+async function enumerateFiles(root, repositoryRoots) {
+    const files = [];
+    let totalBytes = 0;
+    let visitedEntries = 0;
+    async function walk(directory) {
+        const handle = await opendir(directory);
+        for await (const entry of handle) {
+            visitedEntries += 1;
+            if (visitedEntries > MAX_ENTRIES)
+                throw new Error('Corpus exceeds directory entry limit');
+            if (entry.name === '.git' || entry.name === 'node_modules')
+                throw new Error(`Corpus contains forbidden entry: ${entry.name}`);
+            const path = join(directory, entry.name);
+            const info = await lstat(path);
+            if (info.isSymbolicLink())
+                throw new Error(`Corpus contains symlink: ${path}`);
+            if (info.isDirectory()) {
+                await walk(path);
+                continue;
+            }
+            if (!info.isFile())
+                throw new Error(`Corpus contains unsupported entry: ${path}`);
+            if (files.length >= MAX_FILES)
+                throw new Error('Corpus exceeds file limit');
+            totalBytes += info.size;
+            if (totalBytes > MAX_TOTAL_BYTES)
+                throw new Error('Corpus exceeds byte limit');
+            const content = await readFile(path);
+            const digest = createHash('sha256').update(content).digest('hex');
+            const rel = relative(root, path).split(sep).join('/');
+            files.push({
+                path: rel,
+                mode: info.mode & 0o777,
+                size: info.size,
+                digest,
+            });
+        }
+    }
+    for (const repositoryRoot of repositoryRoots)
+        await walk(repositoryRoot);
+    return files;
+}
+function contentDigest(files) {
+    const records = [...files]
+        .sort((a, b) => compareStrings(a.path, b.path))
+        .map((file) => `${file.path}\0${file.mode.toString(8)}\0${file.size}\0${file.digest}\n`)
+        .join('');
+    return `sha256:${createHash('sha256').update(records, 'utf8').digest('hex')}`;
+}
+export async function loadHeldOutCorpus(root) {
+    const corpusRoot = await realpath(resolve(root));
+    const manifestPath = join(corpusRoot, 'corpus.yaml');
+    const manifestInfo = await lstat(manifestPath);
+    if (manifestInfo.isSymbolicLink() || !manifestInfo.isFile())
+        throw new Error('corpus.yaml must be a regular file');
+    if (manifestInfo.size > MAX_MANIFEST_BYTES)
+        throw new Error('corpus.yaml exceeds 1 MiB');
+    const raw = parseYaml(await readFile(manifestPath, 'utf8'), {
+        maxAliasCount: 20,
+    });
+    bounded(raw);
+    const manifest = object(raw, 'corpus');
+    exactKeys(manifest, [
+        'schema_version',
+        'id',
+        'version',
+        'split',
+        'repetitions',
+        'policy',
+        'content_sha256',
+        'repositories',
+    ], 'corpus');
+    if (manifest.schema_version !== '1.0')
+        throw new Error('Unsupported corpus schema_version');
+    if (manifest.id !== 'praxis-held-out-v1')
+        throw new Error('Unsupported corpus id');
+    if (manifest.version !== 1)
+        throw new Error('Unsupported corpus version');
+    if (manifest.split !== 'held-out')
+        throw new Error('corpus split must be held-out');
+    if (manifest.repetitions !== 3)
+        throw new Error('corpus repetitions must be 3');
+    const policyRaw = object(manifest.policy, 'corpus.policy');
+    exactKeys(policyRaw, ['execution', 'tuning', 'result_informed_changes'], 'corpus.policy');
+    if (policyRaw.execution !== 'opt-in-only' ||
+        policyRaw.tuning !== 'forbidden' ||
+        policyRaw.result_informed_changes !== 'require-new-version')
+        throw new Error('corpus policy is invalid');
+    const declaredDigest = text(manifest.content_sha256, 'corpus.content_sha256');
+    if (!/^sha256:[0-9a-f]{64}$/u.test(declaredDigest))
+        throw new Error('corpus.content_sha256 is invalid');
+    if (!Array.isArray(manifest.repositories) || manifest.repositories.length < 3)
+        throw new Error('corpus must declare at least three repositories');
+    const repositories = [];
+    const ids = new Set();
+    const targets = [];
+    const globalTasks = new Set();
+    for (const [index, value] of manifest.repositories.entries()) {
+        const item = object(value, `repositories[${index}]`);
+        exactKeys(item, ['id', 'path', 'tasks'], `repositories[${index}]`);
+        const id = text(item.id, `repositories[${index}].id`);
+        if (!/^[a-z0-9]+(?:-[a-z0-9]+)*$/u.test(id) || ids.has(id))
+            throw new Error('repository IDs must be unique lowercase dash-delimited identifiers');
+        ids.add(id);
+        const path = safeRelativePath(item.path, `repositories[${index}].path`);
+        const tasksRaw = item.tasks;
+        if (!Array.isArray(tasksRaw) ||
+            tasksRaw.length < 4 ||
+            tasksRaw.some((task) => typeof task !== 'string'))
+            throw new Error(`repositories[${index}].tasks must contain at least four task names`);
+        const tasks = tasksRaw.map((task) => text(task, `repositories[${index}].tasks`));
+        if (new Set(tasks).size !== tasks.length ||
+            [...tasks].sort(compareStrings).some((task, i) => task !== tasks[i]))
+            throw new Error(`repositories[${index}].tasks must be unique and sorted`);
+        for (const task of tasks) {
+            if (globalTasks.has(task))
+                throw new Error(`Task name is duplicated across repositories: ${task}`);
+            globalTasks.add(task);
+        }
+        const target = await regularContainedPath(corpusRoot, path, `repositories[${index}].path`);
+        targets.push(target);
+        repositories.push({ id, path, target, tasks, cases: [] });
+    }
+    if (globalTasks.size < 12)
+        throw new Error('corpus must contain at least twelve tasks');
+    for (let i = 0; i < targets.length; i += 1) {
+        for (let j = i + 1; j < targets.length; j += 1) {
+            const left = targets[i];
+            const right = targets[j];
+            if (left === undefined || right === undefined)
+                continue;
+            if (left === right ||
+                left.startsWith(`${right}${sep}`) ||
+                right.startsWith(`${left}${sep}`))
+                throw new Error('repository roots must be unique and non-nested');
+        }
+    }
+    // Preflight every repository before Project Eval discovery. This bounds the
+    // tree before the existing discovery walker inspects any case definitions.
+    const files = await enumerateFiles(corpusRoot, targets);
+    const finalRepositories = [];
+    for (const repository of repositories) {
+        const cases = await discoverProjectEvalCases(repository.target);
+        cases.sort((left, right) => compareStrings(left.name, right.name));
+        const names = cases.map((item) => item.name);
+        if (names.length !== repository.tasks.length ||
+            names.some((name, index) => name !== repository.tasks[index]))
+            throw new Error(`Repository ${repository.id} task declaration does not match discovery`);
+        for (const item of cases) {
+            if (item.runs !== 3)
+                throw new Error(`${item.name} must have three repetitions`);
+            if (!item.verification.some((verifier) => verifier.required))
+                throw new Error(`${item.name} must have a required verifier`);
+            if (!item.expect.allowedChangedPaths.length ||
+                !item.expect.expectedChangedPaths.length ||
+                !item.expect.forbiddenChangedPaths.length)
+                throw new Error(`${item.name} must declare allowed, expected, and forbidden mutations`);
+            if (item.execution.model !== undefined)
+                throw new Error(`${item.name} must not pin a model`);
+            const allowed = new Set(item.expect.allowedChangedPaths);
+            const expected = new Set(item.expect.expectedChangedPaths);
+            const mutationPaths = [...allowed, ...expected];
+            if (mutationPaths.some((path) => new Minimatch(path, MUTATION_GLOB_OPTIONS).hasMagic()))
+                throw new Error(`${item.name} allowed and expected mutations must use exact paths`);
+            if ([...expected].some((path) => !allowed.has(path)))
+                throw new Error(`${item.name} expected mutations must be allowed`);
+            const forbidden = item.expect.forbiddenChangedPaths.map((pattern) => new Minimatch(pattern, MUTATION_GLOB_OPTIONS));
+            if (mutationPaths.some((path) => forbidden.some((matcher) => matcher.match(path))))
+                throw new Error(`${item.name} mutation paths overlap`);
+            if (!REQUIRED_TAGS.every((tag) => item.tags.includes(tag)) ||
+                !item.tags.includes(repository.id))
+                throw new Error(`${item.name} is missing required held-out tags`);
+            if (item.tags.some((tag) => FORBIDDEN_TAGS.has(tag)))
+                throw new Error(`${item.name} contains a forbidden tuning tag`);
+        }
+        finalRepositories.push({ ...repository, cases });
+    }
+    const actualDigest = contentDigest(files);
+    if (actualDigest !== declaredDigest)
+        throw new Error(`corpus content digest mismatch: expected ${declaredDigest}, got ${actualDigest}`);
+    const taskCount = globalTasks.size;
+    return {
+        root: corpusRoot,
+        schemaVersion: '1.0',
+        id: 'praxis-held-out-v1',
+        version: 1,
+        split: 'held-out',
+        repetitions: 3,
+        policy: {
+            execution: 'opt-in-only',
+            tuning: 'forbidden',
+            resultInformedChanges: 'require-new-version',
+        },
+        contentSha256: actualDigest,
+        repositories: finalRepositories,
+        taskCount,
+        plannedRunCount: taskCount * 3,
+    };
+}
+//# sourceMappingURL=held-out-corpus.js.map

package/dist/evals/held-out-qualification.d.ts

@@ -0,0 +1,140 @@
+import { type PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
+import { type ProjectEvalIdentity } from './project-eval-identity.js';
+import { type ProjectEvalComparisonMetric } from './project-eval-comparison.js';
+import type { ProjectEvalDependencies, ProjectEvalRunSummary } from './project-eval.js';
+/**
+ * Qualification orchestration for the immutable held-out corpus.
+ *
+ * This module owns the qualification contract and evidence envelope; actual
+ * case execution remains in the Project Eval runner.
+ */
+export declare const HELD_OUT_QUALIFICATION_HELP = "Usage: praxis eval qualify [options] <corpus>\n\nQualify an explicitly pinned provider/model against the immutable held-out corpus.\n\nOptions:\n\nRequired:\n  --provider <id>             Provider identifier\n  --profile <id>              Provider profile identifier\n  --model <id>                Model identifier\n  --confirm-held-out <id@sha256>  Confirm the exact held-out corpus digest\n  --run-verification          Run the declared case verifiers\n  --output-dir <dir>          Write qualification artifacts here\n  --allow-tools <rules>       Grant gated tools; comma-separated and repeatable\n\nOptional:\n  --baseline <qualification-result.json>  Compare against a completed result\n  --keep-temp                 Preserve temporary workspaces\n  --json                      Print exactly one qualification JSON value\n  --verbose                   Print run progress to stderr\n  -h, --help                  Display help";
+export interface HeldOutQualificationOptions {
+    corpus?: string;
+    provider?: string;
+    profile?: string;
+    model?: string;
+    confirmHeldOut?: string;
+    allowTools: string[];
+    runVerification: boolean;
+    outputDir?: string;
+    baseline?: string;
+    keepTemp: boolean;
+    json: boolean;
+    verbose: boolean;
+    help?: true;
+}
+/** Parse the strict, side-effect-free qualification command line. */
+export declare function parseHeldOutQualificationOptions(argv: readonly string[]): HeldOutQualificationOptions;
+export interface HeldOutQualificationAggregateReference {
+    repository: string;
+    path: string;
+    sha256: `sha256:${string}`;
+    identity_sha256: `sha256:${string}`;
+}
+export interface HeldOutQualificationRun {
+    repository: string;
+    case: string;
+    run: number;
+    passed: boolean;
+    safety_passed: boolean;
+    verifier_satisfied: boolean;
+    turns: number;
+    duration_ms: number;
+    usage_known: boolean;
+    cost_known: boolean;
+    usage: ProjectEvalRunSummary['usage'];
+    cost_usd: number | null;
+    identity: ProjectEvalIdentity;
+}
+export interface HeldOutQualificationCaseSummary {
+    repository: string;
+    case: string;
+    repetitions: number;
+    passed: number;
+    failed: number;
+    safety_passed: number;
+    safety_failed: number;
+    verifier_satisfied_runs: number;
+    verifier_unsatisfied_runs: number;
+    passed_all: boolean;
+    safety_passed_all: boolean;
+    verifier_satisfied: boolean;
+}
+export interface HeldOutQualificationResult {
+    schema_version: '1.0';
+    corpus: {
+        id: string;
+        version: number;
+        content_sha256: `sha256:${string}`;
+        repository_count: number;
+        task_count: number;
+        repetitions: number;
+        planned_run_count: number;
+    };
+    provider: string;
+    profile: string;
+    protocol: string;
+    model: string;
+    endpoint_sha256: `sha256:${string}`;
+    plan_sha256: `sha256:${string}`;
+    praxis_version: string;
+    build: PraxisBuildIdentity;
+    node_version: string;
+    platform: string;
+    architecture: string;
+    start: string;
+    duration_ms: number;
+    aggregates: readonly HeldOutQualificationAggregateReference[];
+    planned_run_count: number;
+    completed_run_count: number;
+    passed: number;
+    failed: number;
+    safety_passed: number;
+    safety_failed: number;
+    verifier_satisfied_runs: number;
+    verifier_unsatisfied_runs: number;
+    usage_known_runs: number;
+    usage_unknown_runs: number;
+    cost_known_runs: number;
+    cost_unknown_runs: number;
+    usage_totals: {
+        input_tokens: number;
+        output_tokens: number;
+        cache_read_input_tokens: number;
+        cache_creation_input_tokens: number;
+        web_search_requests: number;
+    } | null;
+    known_cost_total_usd: number | null;
+    median_turns: number;
+    p95_turns: number;
+    median_duration_ms: number;
+    p95_duration_ms: number;
+    cases: readonly HeldOutQualificationCaseSummary[];
+    runs: readonly HeldOutQualificationRun[];
+    baseline?: {
+        source_sha256: `sha256:${string}`;
+        summary: {
+            passed: number;
+            failed: number;
+            pass_rate: number;
+            safety_pass_rate: number;
+            planned_run_count: number;
+        };
+    };
+    regressions: readonly {
+        case: string;
+        run: number;
+        baseline_passed: boolean;
+        candidate_passed: boolean;
+    }[];
+    metric_deltas: Readonly<Record<string, ProjectEvalComparisonMetric<number | null>>>;
+    qualified: boolean | null;
+    optimization_claim_allowed: boolean;
+}
+export type HeldOutQualificationDependencies = ProjectEvalDependencies;
+export declare function executeHeldOutQualificationCommand(argv: readonly string[], io: {
+    stdout(message: string): void;
+    stderr(message: string): void;
+}, dependencies: HeldOutQualificationDependencies, callerCwd?: string, signal?: AbortSignal): Promise<number>;
+//# sourceMappingURL=held-out-qualification.d.ts.map