praxis-agent 0.65.0 → 0.66.0

package/README.md

@@ -134,12 +134,16 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
 
 - **Outcome-driven evaluation** — `praxis eval <target>` runs contained cases
   in isolated workspaces, requires explicit verifier authorization, and writes
-  versioned artifacts with deterministic run identities locally; usage and cost
+  versioned artifacts with deterministic run identities locally, including the
+  Praxis source revision, dirty state, and emitted-runtime digest; usage and cost
   remain explicitly available or unknown. Separate runs can be compared with
   `praxis eval compare` only when provider, model, configuration, tools, prompt,
-  corpus, and runtime-environment identities match. Unknown token/cost evidence
-  produces null deltas, while the gate requires no pass-rate or safety-rate
-  regression and rejects incomplete safety evidence.
+  corpus, and host-runtime identities match; Praxis version and build provenance
+  remain attribution fields so candidate builds may differ. Unknown token/cost
+  evidence produces null deltas, while the gate requires no pass-rate or
+  safety-rate regression, every candidate verifier to be satisfied, and every
+  candidate `high` or `release` task to pass, and rejects incomplete safety
+  evidence.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,
@@ -403,9 +407,10 @@ normal/low-capability full-frame p95 budgets of `<16.7/<33 ms`.
 `npm run test:coverage` measures all production code under `src/**` with V8 and
 enforces global floors of 79% statements, 70% branches, 85% functions, and 81% lines,
 and rejects any production runtime module with zero covered statements (while allowing
-type-only modules). `npm run test:fixtures` executes the 71-behavior native contract; 63 behaviors
-are qualified and 8 are explicitly excluded. `npm run verify:fixture-contracts`
-performs the structural check and is part of `npm run check`.
+type-only modules). `npm run test:fixtures` executes the 74-behavior native contract; 66 behaviors
+are qualified and 8 are explicitly excluded. Schema-v2 risk tiers and executable evidence dimensions
+are enforced fail-closed. `npm run verify:fixture-contracts` performs the structural check and is part
+of `npm run check`.
 `npm run test:core-completion` is retained as a compatibility alias for
 `npm run test:fixtures`.
 

package/dist/build-identity.json

@@ -0,0 +1 @@
+{"schema_version":"1.0","source_revision":"git:8ed6222ee3a981ea0a00bb484ad43e13e278ab5d","source_dirty":false,"artifact_sha256":"sha256:8eb26913c75e9288f8387f5af1b5aefe847942e4c1d35598130cdac53d758524"}

package/dist/cli-runtime.js

@@ -73,6 +73,7 @@ import { addClaudeMarketplace, disableAllNativePlugins, installClaudeMarketplace
 import { executeClaudePluginEvalCommand, PLUGIN_EVAL_HELP, } from './plugins/claude-plugin-eval.js';
 import { executeProjectEvalCommand, PROJECT_EVAL_HELP, } from './evals/project-eval.js';
 import { PROJECT_EVAL_COMPARE_HELP } from './evals/project-eval-comparison.js';
+import { loadPraxisBuildIdentity } from './platform/praxis-build-identity.js';
 import { CLAUDE_PLUGIN_PRUNE_HELP, CLAUDE_PLUGIN_TAG_HELP, executeClaudePluginPrune, planClaudePluginPrune, tagClaudePlugin, } from './plugins/claude-plugin-maintenance.js';
 import { formatDoctorReport, runDoctor } from './maintenance/doctor.js';
 import { runSelfUpdate, } from './maintenance/self-update.js';
@@ -2622,6 +2623,7 @@ export function createDefaultDependencies(entrypoint = fileURLToPath(import.meta
         },
         projectEval: {
             runtimeFactory: defaultProjectEvalRuntimeFactory,
+            loadBuildIdentity: () => loadPraxisBuildIdentity(),
             version: VERSION,
             configRoot: resolveDataPlaneRoot(),
         },

package/dist/evals/project-eval-comparison.d.ts

@@ -1,4 +1,5 @@
 import type { ProjectEvalAggregate } from './project-eval.js';
+import { type ProjectEvalRisk } from './project-eval-schema.js';
 export declare const PROJECT_EVAL_COMPARE_HELP = "Usage: praxis eval compare [options]\n\nCompare two completed project evaluation aggregate artifacts.\n\nOptions:\n  --baseline <aggregate-result.json>   Baseline aggregate artifact\n  --baseline-name <name>               Name shown for the baseline\n  --candidate <aggregate-result.json>  Candidate aggregate artifact\n  --candidate-name <name>              Name shown for the candidate\n  --output-dir <dir>                   Write comparison-result.json here\n  --json                               Print exactly one comparison JSON value\n  -h, --help                           Display help";
 export interface ProjectEvalCompareOptions {
     baseline?: string;
@@ -20,7 +21,7 @@ export interface ProjectEvalComparisonMetric<T = number | null> {
     delta: T;
 }
 export interface ProjectEvalComparisonResult {
-    schema_version: '1.1';
+    schema_version: '1.2';
     baseline: {
         name: string;
         source_path: string;
@@ -45,6 +46,8 @@ export interface ProjectEvalComparisonResult {
     }[];
     metrics: {
         pass_rate: ProjectEvalComparisonMetric<number>;
+        verification_pass_rate: ProjectEvalComparisonMetric<number | null>;
+        task_risk_pass_rate: Record<ProjectEvalRisk, ProjectEvalComparisonMetric<number | null>>;
         safety_pass_rate: ProjectEvalComparisonMetric<number | null>;
         average_turns: ProjectEvalComparisonMetric<number>;
         input_tokens: ProjectEvalComparisonMetric<number | null>;

package/dist/evals/project-eval-comparison.js

@@ -2,6 +2,8 @@ import { mkdir, lstat, readFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { writeFileAtomically } from '../platform/atomic-write.js';
 import { assertProjectEvalIdentitiesComparable, validateProjectEvalAggregateIdentity, validateProjectEvalIdentity, } from './project-eval-identity.js';
+import { PROJECT_EVAL_SAFETY_CHECKS } from './project-eval-runner.js';
+import { PROJECT_EVAL_MAX_ITEMS, PROJECT_EVAL_RISKS, } from './project-eval-schema.js';
 const MAX_AGGREGATE_BYTES = 8 * 1024 * 1024;
 const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
 export const PROJECT_EVAL_COMPARE_HELP = `Usage: praxis eval compare [options]
@@ -107,12 +109,6 @@ function objectField(value, path) {
         fail(path, 'expected an object');
     return value;
 }
-function optionalGroup(value, keys, path) {
-    const present = keys.filter((key) => value[key] !== undefined);
-    if (present.length !== 0 && present.length !== keys.length)
-        fail(path, `fields must be all present or all absent: ${keys.join(', ')}`);
-    return present.length === keys.length;
-}
 function usageField(value, path) {
     if (value === null)
         return null;
@@ -131,9 +127,86 @@ function usageField(value, path) {
         fail(path, 'inputTokens and outputTokens are required');
     return usage;
 }
+const FIXED_CHECKS = [
+    'runtime',
+    'termination',
+    ...PROJECT_EVAL_SAFETY_CHECKS,
+    'expected-paths',
+];
+const MAX_COMPACT_CHECKS = FIXED_CHECKS.length + 2 * PROJECT_EVAL_MAX_ITEMS;
+function checkEvidence(value, path) {
+    if (!Array.isArray(value) || value.length > MAX_COMPACT_CHECKS)
+        fail(path, 'expected a bounded array');
+    const names = new Set();
+    const checks = value.map((item, index) => {
+        const check = objectField(item, `${path}[${index}]`);
+        const unknown = Object.keys(check).find((key) => !['name', 'passed'].includes(key));
+        if (unknown)
+            fail(`${path}[${index}]`, `unknown field: ${unknown}`);
+        const name = stringField(check.name, `${path}[${index}].name`);
+        boolField(check.passed, `${path}[${index}].passed`);
+        if (names.has(name))
+            fail(path, 'check names must be unique');
+        names.add(name);
+        const fixed = FIXED_CHECKS.includes(name);
+        const dynamic = name === 'graders' ||
+            /^verifier:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(name) ||
+            /^grader:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(name);
+        if (!fixed && !dynamic)
+            fail(`${path}[${index}].name`, 'unsupported check');
+        return { name, passed: check.passed };
+    });
+    for (const name of FIXED_CHECKS)
+        if (!names.has(name))
+            fail(path, `missing required check: ${name}`);
+    return checks;
+}
+function verificationEvidence(value, path) {
+    const evidence = objectField(value, path);
+    const unknownEvidence = Object.keys(evidence).find((key) => !['outcomes', 'satisfied'].includes(key));
+    if (unknownEvidence)
+        fail(path, `unknown field: ${unknownEvidence}`);
+    const raw = evidence.outcomes;
+    if (!Array.isArray(raw) || raw.length > 256)
+        fail(`${path}.outcomes`, 'expected a bounded array');
+    const names = new Set();
+    const outcomes = raw.map((item, index) => {
+        const outcome = objectField(item, `${path}.outcomes[${index}]`);
+        const unknown = Object.keys(outcome).find((key) => !['name', 'required', 'expect', 'status'].includes(key));
+        if (unknown)
+            fail(`${path}.outcomes[${index}]`, `unknown field: ${unknown}`);
+        const name = stringField(outcome.name, `${path}.outcomes[${index}].name`);
+        if (!IDENTIFIER.test(name))
+            fail(`${path}.outcomes[${index}].name`, 'unsafe verifier name');
+        if (names.has(name))
+            fail(`${path}.outcomes`, 'verifier names must be unique');
+        names.add(name);
+        if (outcome.required !== true)
+            fail(`${path}.outcomes[${index}].required`, 'must be true');
+        if (outcome.expect !== 'pass')
+            fail(`${path}.outcomes[${index}].expect`, 'must be pass');
+        if (outcome.status !== 'passed' &&
+            outcome.status !== 'failed' &&
+            outcome.status !== 'not_run')
+            fail(`${path}.outcomes[${index}].status`, 'invalid verifier status');
+        return {
+            name,
+            required: true,
+            expect: 'pass',
+            status: outcome.status,
+        };
+    });
+    boolField(evidence.satisfied, `${path}.satisfied`);
+    const satisfied = outcomes.every((outcome) => outcome.status === 'passed');
+    if (evidence.satisfied !== satisfied)
+        fail(`${path}.satisfied`, 'does not match verifier statuses');
+    return { outcomes, satisfied };
+}
 function validateRun(value, index) {
     const path = `runs[${index}]`;
     const run = objectField(value, path);
+    if (run.schema_version !== '1.2')
+        fail(`${path}.schema_version`, 'must be "1.2"');
     stringField(run.case, `${path}.case`);
     if (!IDENTIFIER.test(run.case))
         fail(`${path}.case`, 'unsafe case name');
@@ -144,7 +217,27 @@ function validateRun(value, index) {
     const identity = validateProjectEvalIdentity(run.identity);
     if (model !== identity.model_id)
         fail(`${path}.model`, 'does not match identity.model_id');
+    if (!PROJECT_EVAL_RISKS.includes(run.risk))
+        fail(`${path}.risk`, 'invalid risk');
+    const checks = checkEvidence(run.checks, `${path}.checks`);
+    const verification = verificationEvidence(run.verification, `${path}.verification`);
+    const verifierChecks = new Map(checks
+        .filter(({ name }) => name.startsWith('verifier:'))
+        .map((check) => [check.name.slice('verifier:'.length), check.passed]));
+    if (verifierChecks.size !== verification.outcomes.length)
+        fail(`${path}.checks`, 'verifier checks do not match outcomes');
+    for (const outcome of verification.outcomes) {
+        const checkPassed = verifierChecks.get(outcome.name);
+        if (checkPassed === undefined)
+            fail(`${path}.verification`, 'missing verifier check');
+        if (checkPassed !== (outcome.status === 'passed'))
+            fail(`${path}.verification`, 'verifier check does not match status');
+    }
+    const recomputedPassed = checks.every((check) => check.passed);
+    const safetyPassed = PROJECT_EVAL_SAFETY_CHECKS.every((name) => checks.find((check) => check.name === name)?.passed === true);
     boolField(run.passed, `${path}.passed`);
+    if (run.passed !== recomputedPassed)
+        fail(`${path}.passed`, 'does not match compact checks');
     if (run.score !== 0 && run.score !== 1)
         fail(`${path}.score`, 'must be 0 or 1');
     if (run.score !== (run.passed ? 1 : 0))
@@ -162,16 +255,24 @@ function validateRun(value, index) {
         fail(`${path}.termination`, 'invalid termination');
     stringField(run.error, `${path}.error`, true);
     stringField(run.artifact_dir, `${path}.artifact_dir`);
-    const evidenceKnown = optionalGroup(run, ['safety_passed', 'permission_decisions', 'tool_errors', 'retries'], path);
-    if (evidenceKnown) {
-        boolField(run.safety_passed, `${path}.safety_passed`);
-        const permissions = objectField(run.permission_decisions, `${path}.permission_decisions`);
-        for (const key of ['allow', 'ask', 'deny'])
-            numberField(permissions[key], `${path}.permission_decisions.${key}`, true);
-        numberField(run.tool_errors, `${path}.tool_errors`, true);
-        numberField(run.retries, `${path}.retries`, true);
-    }
-    return { ...run, model, identity };
+    boolField(run.safety_passed, `${path}.safety_passed`);
+    if (run.safety_passed !== safetyPassed)
+        fail(`${path}.safety_passed`, 'does not match safety checks');
+    const permissions = objectField(run.permission_decisions, `${path}.permission_decisions`);
+    for (const key of ['allow', 'ask', 'deny'])
+        numberField(permissions[key], `${path}.permission_decisions.${key}`, true);
+    numberField(run.tool_errors, `${path}.tool_errors`, true);
+    numberField(run.retries, `${path}.retries`, true);
+    const terminationCheck = checks.find((check) => check.name === 'termination');
+    if (terminationCheck?.passed !== (run.termination === null))
+        fail(`${path}.checks`, 'termination check does not match termination');
+    return {
+        ...run,
+        model,
+        identity,
+        checks,
+        verification,
+    };
 }
 export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cwd()) {
     const sourcePath = resolve(callerCwd, inputPath);
@@ -194,8 +295,8 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     }
     const aggregate = objectField(value, 'root');
     const data = aggregate;
-    if (aggregate.schema_version !== '1.1')
-        fail('schema_version', 'must be "1.1"; legacy "1.0" aggregates are unsupported');
+    if (aggregate.schema_version !== '1.2')
+        fail('schema_version', 'must be "1.2"; legacy "1.1" aggregates are unsupported');
     stringField(aggregate.version, 'version');
     stringField(aggregate.start, 'start');
     numberField(aggregate.duration_ms, 'duration_ms');
@@ -311,64 +412,95 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     if (data.partial !==
         (data.interrupted || data.run_count < data.planned_run_count))
         fail('partial', 'does not match interrupted/completed run state');
-    const aggregateEvidenceKnown = optionalGroup(aggregate, [
-        'safety_passed',
-        'safety_failed',
-        'permission_decisions',
-        'tool_errors',
-        'retries',
-        'terminations',
-    ], 'root evidence');
-    const runEvidenceKnown = runs.every((run) => typeof run.safety_passed === 'boolean' &&
-        typeof run.tool_errors === 'number' &&
-        typeof run.retries === 'number' &&
-        run.permission_decisions !== undefined);
-    const runEvidenceAbsent = runs.every((run) => run.safety_passed === undefined &&
-        run.tool_errors === undefined &&
-        run.retries === undefined &&
-        run.permission_decisions === undefined);
-    if ((aggregateEvidenceKnown && !runEvidenceKnown) ||
-        (!aggregateEvidenceKnown && !runEvidenceAbsent))
-        fail('runs', 'run evidence must match aggregate evidence availability');
-    const safetyKnown = aggregateEvidenceKnown && runEvidenceKnown;
-    if (aggregateEvidenceKnown) {
-        numberField(aggregate.safety_passed, 'safety_passed', true);
-        numberField(aggregate.safety_failed, 'safety_failed', true);
-        const permissions = objectField(aggregate.permission_decisions, 'permission_decisions');
-        for (const key of ['allow', 'ask', 'deny'])
-            numberField(permissions[key], `permission_decisions.${key}`, true);
-        numberField(aggregate.tool_errors, 'tool_errors', true);
-        numberField(aggregate.retries, 'retries', true);
-        if (aggregate.safety_passed !==
-            runs.filter((run) => run.safety_passed).length ||
-            aggregate.safety_failed !==
-                runs.filter((run) => !run.safety_passed).length)
-            fail('safety_passed', 'does not match run safety evidence');
-        const expectedPermissions = {
-            allow: runs.reduce((total, run) => total + run.permission_decisions.allow, 0),
-            ask: runs.reduce((total, run) => total + run.permission_decisions.ask, 0),
-            deny: runs.reduce((total, run) => total + run.permission_decisions.deny, 0),
-        };
-        if (permissions.allow !== expectedPermissions.allow ||
-            permissions.ask !== expectedPermissions.ask ||
-            permissions.deny !== expectedPermissions.deny)
-            fail('permission_decisions', 'does not match run evidence');
-        if (aggregate.tool_errors !==
-            runs.reduce((total, run) => total + run.tool_errors, 0) ||
-            aggregate.retries !== runs.reduce((total, run) => total + run.retries, 0))
-            fail('tool_errors', 'does not match run evidence');
-        const terminations = objectField(aggregate.terminations, 'terminations');
-        for (const key of ['completed', 'timeout', 'interrupted'])
-            numberField(terminations[key], `terminations.${key}`, true);
-        const expectedTerminations = {
-            completed: runs.filter((run) => run.termination === null).length,
-            timeout: runs.filter((run) => run.termination === 'timeout').length,
-            interrupted: runs.filter((run) => run.termination === 'interrupted')
-                .length,
-        };
-        for (const key of Object.keys(expectedTerminations))
-            if (terminations[key] !== expectedTerminations[key])
-                fail(`terminations.${key}`, 'does not match runs');
+    const safetyKnown = true;
+    numberField(aggregate.safety_passed, 'safety_passed', true);
+    numberField(aggregate.safety_failed, 'safety_failed', true);
+    const permissions = objectField(aggregate.permission_decisions, 'permission_decisions');
+    for (const key of ['allow', 'ask', 'deny'])
+        numberField(permissions[key], `permission_decisions.${key}`, true);
+    numberField(aggregate.tool_errors, 'tool_errors', true);
+    numberField(aggregate.retries, 'retries', true);
+    if (aggregate.safety_passed !==
+        runs.filter((run) => run.safety_passed).length ||
+        aggregate.safety_failed !== runs.filter((run) => !run.safety_passed).length)
+        fail('safety_passed', 'does not match run safety evidence');
+    const expectedPermissions = {
+        allow: runs.reduce((total, run) => total + run.permission_decisions.allow, 0),
+        ask: runs.reduce((total, run) => total + run.permission_decisions.ask, 0),
+        deny: runs.reduce((total, run) => total + run.permission_decisions.deny, 0),
+    };
+    if (permissions.allow !== expectedPermissions.allow ||
+        permissions.ask !== expectedPermissions.ask ||
+        permissions.deny !== expectedPermissions.deny)
+        fail('permission_decisions', 'does not match run evidence');
+    if (aggregate.tool_errors !==
+        runs.reduce((total, run) => total + run.tool_errors, 0) ||
+        aggregate.retries !== runs.reduce((total, run) => total + run.retries, 0))
+        fail('tool_errors', 'does not match run evidence');
+    const terminations = objectField(aggregate.terminations, 'terminations');
+    for (const key of ['completed', 'timeout', 'interrupted'])
+        numberField(terminations[key], `terminations.${key}`, true);
+    const expectedTerminations = {
+        completed: runs.filter((run) => run.termination === null).length,
+        timeout: runs.filter((run) => run.termination === 'timeout').length,
+        interrupted: runs.filter((run) => run.termination === 'interrupted').length,
+    };
+    for (const key of Object.keys(expectedTerminations))
+        if (terminations[key] !== expectedTerminations[key])
+            fail(`terminations.${key}`, 'does not match runs');
+    const verificationTotals = objectField(aggregate.verification_totals, 'verification_totals');
+    const unknownVerificationTotal = Object.keys(verificationTotals).find((key) => ![
+        'declared',
+        'passed',
+        'failed',
+        'not_run',
+        'satisfied_runs',
+        'unsatisfied_runs',
+    ].includes(key));
+    if (unknownVerificationTotal)
+        fail('verification_totals', `unknown field: ${unknownVerificationTotal}`);
+    for (const key of [
+        'declared',
+        'passed',
+        'failed',
+        'not_run',
+        'satisfied_runs',
+        'unsatisfied_runs',
+    ])
+        numberField(verificationTotals[key], `verification_totals.${key}`, true);
+    const expectedVerificationTotals = {
+        declared: runs.reduce((total, run) => total + run.verification.outcomes.length, 0),
+        passed: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'passed')
+                .length, 0),
+        failed: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'failed')
+                .length, 0),
+        not_run: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'not_run')
+                .length, 0),
+        satisfied_runs: runs.filter((run) => run.verification.satisfied).length,
+        unsatisfied_runs: runs.filter((run) => !run.verification.satisfied).length,
+    };
+    for (const key of Object.keys(expectedVerificationTotals))
+        if (verificationTotals[key] !== expectedVerificationTotals[key])
+            fail(`verification_totals.${key}`, 'does not match run verification evidence');
+    const riskTiers = objectField(aggregate.risk_tiers, 'risk_tiers');
+    const unknownRisk = Object.keys(riskTiers).find((key) => !PROJECT_EVAL_RISKS.includes(key));
+    if (unknownRisk)
+        fail('risk_tiers', `unknown field: ${unknownRisk}`);
+    for (const risk of PROJECT_EVAL_RISKS) {
+        const tier = objectField(riskTiers[risk], `risk_tiers.${risk}`);
+        const unknownTier = Object.keys(tier).find((key) => !['runs', 'passed', 'failed'].includes(key));
+        if (unknownTier)
+            fail(`risk_tiers.${risk}`, `unknown field: ${unknownTier}`);
+        for (const key of ['runs', 'passed', 'failed'])
+            numberField(tier[key], `risk_tiers.${risk}.${key}`, true);
+        const riskRuns = runs.filter((run) => run.risk === risk);
+        if (tier.runs !== riskRuns.length ||
+            tier.passed !== riskRuns.filter((run) => run.passed).length ||
+            tier.failed !== riskRuns.filter((run) => !run.passed).length)
+            fail(`risk_tiers.${risk}`, 'does not match run risk evidence');
     }
     return {
         aggregate: { ...aggregate, runs },
@@ -434,6 +566,21 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
     const rightSafety = safetyKnown
         ? rightRuns.filter((run) => run.safety_passed).length / rightRuns.length
         : null;
+    const verificationRate = (runs) => {
+        const declared = runs.reduce((total, run) => total + run.verification.outcomes.length, 0);
+        if (declared === 0)
+            return null;
+        const passed = runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'passed')
+                .length, 0);
+        return passed / declared;
+    };
+    const taskRiskRate = (runs, risk) => {
+        const selected = runs.filter((run) => run.risk === risk);
+        return selected.length === 0
+            ? null
+            : selected.filter((run) => run.passed).length / selected.length;
+    };
     const terms = (runs, kind) => runs.filter((run) => kind === 'completed'
         ? run.termination === null
         : run.termination === kind).length;
@@ -442,7 +589,7 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         ? metric(leftRuns.reduce((n, r) => n + r.permission_decisions[name], 0), rightRuns.reduce((n, r) => n + r.permission_decisions[name], 0))
         : nullableMetric(null, null);
     const result = {
-        schema_version: '1.1',
+        schema_version: '1.2',
         baseline: {
             name: baselineName,
             source_path: baseline.sourcePath,
@@ -461,10 +608,19 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         passed: regressions.length === 0 &&
             right.pass_rate >= left.pass_rate &&
             safetyKnown &&
-            (rightSafety ?? 0) >= (leftSafety ?? 0),
+            (rightSafety ?? 0) >= (leftSafety ?? 0) &&
+            rightRuns.every((run) => run.verification.satisfied) &&
+            rightRuns
+                .filter((run) => run.risk === 'high' || run.risk === 'release')
+                .every((run) => run.passed),
         regressions,
         metrics: {
             pass_rate: metric(left.pass_rate, right.pass_rate),
+            verification_pass_rate: nullableMetric(verificationRate(leftRuns), verificationRate(rightRuns)),
+            task_risk_pass_rate: Object.fromEntries(PROJECT_EVAL_RISKS.map((risk) => [
+                risk,
+                nullableMetric(taskRiskRate(leftRuns, risk), taskRiskRate(rightRuns, risk)),
+            ])),
             safety_pass_rate: nullableMetric(leftSafety, rightSafety),
             average_turns: metric(avg(leftRuns.map((r) => r.turns)), avg(rightRuns.map((r) => r.turns))),
             input_tokens: token('input_tokens'),

package/dist/evals/project-eval-identity.d.ts

@@ -1,7 +1,8 @@
 import type { EvalRuntimeIdentityDescriptor } from './eval-contract.js';
 import type { ProjectEvalCase } from './project-eval-schema.js';
 import type { FileManifest } from './project-eval-workspace.js';
-export declare const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION: "1.0";
+import { type PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
+export declare const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION: "1.1";
 export type ProjectEvalIdentitySchemaVersion = typeof PROJECT_EVAL_IDENTITY_SCHEMA_VERSION;
 export type IdentityDigest = `sha256:${string}`;
 export interface ProjectEvalRuntimeIdentity {
@@ -10,6 +11,7 @@ export interface ProjectEvalRuntimeIdentity {
     node_version: string;
     platform: string;
     architecture: string;
+    build: PraxisBuildIdentity;
     runtime_sha256: IdentityDigest;
 }
 export interface ProjectEvalIdentity {
@@ -36,6 +38,7 @@ export interface CreateProjectEvalIdentityInput {
     nodeVersion?: string;
     platform?: string;
     architecture?: string;
+    buildIdentity: PraxisBuildIdentity;
 }
 export interface AggregateIdentityRun {
     case: string;

package/dist/evals/project-eval-identity.js

@@ -1,7 +1,8 @@
 import { createHash } from 'node:crypto';
 import { posix, win32 } from 'node:path';
 import { redactSensitiveValue, sensitiveEnvironmentValues, } from '../platform/sensitive-data.js';
-export const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION = '1.0';
+import { validatePraxisBuildIdentity, } from '../platform/praxis-build-identity.js';
+export const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION = '1.1';
 const DIGEST = /^sha256:[0-9a-f]{64}$/u;
 const MAX_DEPTH = 32;
 const MAX_NODES = 500_000;
@@ -122,6 +123,7 @@ function configurationSource(input) {
         case: {
             name: c.name,
             schema_version: c.schemaVersion,
+            risk: c.risk,
         },
         execution: {
             max_turns: c.execution.maxTurns,
@@ -178,12 +180,14 @@ function freeze(value) {
 }
 export function createProjectEvalIdentity(input) {
     const provider = input.provider;
+    const buildIdentity = validatePraxisBuildIdentity(input.buildIdentity);
     const runtimeBase = {
         engine: 'praxis',
         praxis_version: string(input.praxisVersion, 'praxis_version'),
         node_version: string(input.nodeVersion ?? process.version, 'node_version'),
         platform: string(input.platform ?? process.platform, 'platform'),
         architecture: string(input.architecture ?? process.arch, 'architecture'),
+        build: buildIdentity,
     };
     const identityWithoutDigests = {
         schema_version: PROJECT_EVAL_IDENTITY_SCHEMA_VERSION,
@@ -219,6 +223,7 @@ function validateRuntime(value, path) {
         'node_version',
         'platform',
         'architecture',
+        'build',
         'runtime_sha256',
     ];
     const runtimeUnknown = Object.keys(runtime).find((key) => !runtimeKeys.includes(key));
@@ -232,6 +237,7 @@ function validateRuntime(value, path) {
         node_version: string(runtime.node_version, `${path}.node_version`),
         platform: string(runtime.platform, `${path}.platform`),
         architecture: string(runtime.architecture, `${path}.architecture`),
+        build: validatePraxisBuildIdentity(runtime.build),
         runtime_sha256: digestField(runtime.runtime_sha256, `${path}.runtime_sha256`),
     };
     if (validated.runtime_sha256 !==
@@ -241,6 +247,7 @@ function validateRuntime(value, path) {
             node_version: validated.node_version,
             platform: validated.platform,
             architecture: validated.architecture,
+            build: validated.build,
         }))
         fail(`${path}.runtime_sha256 does not match runtime fields`);
     return validated;
@@ -267,7 +274,7 @@ export function validateProjectEvalIdentity(value) {
     if (unknown)
         fail(`${unknown} is not supported`);
     if (source.schema_version !== PROJECT_EVAL_IDENTITY_SCHEMA_VERSION)
-        fail('schema_version must be "1.0"');
+        fail('schema_version must be "1.1"');
     const validated = {
         schema_version: PROJECT_EVAL_IDENTITY_SCHEMA_VERSION,
         provider_id: string(source.provider_id, 'provider_id'),

package/dist/evals/project-eval-runner.d.ts

@@ -2,10 +2,14 @@ import type { ModelUsage } from '../core/runtime.js';
 import { type EvalGraderResult, type IdentifiedEvalRuntimeFactory } from './eval-contract.js';
 import { type ProjectEvalIdentity } from './project-eval-identity.js';
 import type { ProjectEvalCase } from './project-eval-schema.js';
+import type { PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
 export type ProjectEvalTermination = 'timeout' | 'interrupted' | null;
 export interface ProjectEvalVerificationResult {
-    schema_version: '1.0';
+    schema_version: '1.1';
     name: string;
+    required: true;
+    expect: 'pass';
+    status: 'passed' | 'failed' | 'not_run';
     command: string;
     argv: readonly string[];
     exit_code: number | null;
@@ -18,8 +22,9 @@ export interface ProjectEvalVerificationResult {
     error: string | null;
 }
 export interface ProjectEvalRunResult {
-    schema_version: '1.1';
+    schema_version: '1.2';
     case: string;
+    risk: ProjectEvalCase['risk'];
     run: number;
     version: string;
     model: string;
@@ -42,6 +47,7 @@ export interface ProjectEvalRunResult {
     retries: number;
     graders: readonly EvalGraderResult[];
     checks: readonly EvalGraderResult[];
+    verification: ProjectEvalVerificationEvidence;
     artifacts: {
         trace: 'trace.jsonl';
         workspace_diff: 'workspace-diff.json';
@@ -53,6 +59,21 @@ export interface ProjectEvalRunResult {
     cleanup_errors: readonly string[];
     temp_root: string | null;
 }
+export interface ProjectEvalCheckSummary {
+    name: string;
+    passed: boolean;
+}
+export interface ProjectEvalVerifierOutcome {
+    name: string;
+    required: true;
+    expect: 'pass';
+    status: 'passed' | 'failed' | 'not_run';
+}
+export interface ProjectEvalVerificationEvidence {
+    outcomes: readonly ProjectEvalVerifierOutcome[];
+    satisfied: boolean;
+}
+export declare const PROJECT_EVAL_SAFETY_CHECKS: readonly ["trace-bounds", "runtime-close", "workspace-manifest", "source-unchanged", "allowed-paths", "forbidden-paths", "artifact-write", "temp-cleanup"];
 interface ProjectEvalRunOptions {
     case: ProjectEvalCase;
     factory: IdentifiedEvalRuntimeFactory;
@@ -63,6 +84,7 @@ interface ProjectEvalRunOptions {
     runVerification?: boolean;
     outputDir: string;
     version: string;
+    buildIdentity: PraxisBuildIdentity;
     signal?: AbortSignal;
 }
 export declare function runProjectEvalCase(options: ProjectEvalRunOptions): Promise<ProjectEvalRunResult>;