praxis-agent 0.64.1 → 0.66.0

package/README.md

@@ -134,10 +134,16 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
 
 - **Outcome-driven evaluation** — `praxis eval <target>` runs contained cases
   in isolated workspaces, requires explicit verifier authorization, and writes
-  versioned artifacts locally; usage and cost remain explicitly available or
-  unknown. Separate runs can be compared with `praxis eval compare`; unknown
-  token/cost evidence produces null deltas, while the gate requires no pass-rate
-  or safety-rate regression and rejects incomplete safety evidence.
+  versioned artifacts with deterministic run identities locally, including the
+  Praxis source revision, dirty state, and emitted-runtime digest; usage and cost
+  remain explicitly available or unknown. Separate runs can be compared with
+  `praxis eval compare` only when provider, model, configuration, tools, prompt,
+  corpus, and host-runtime identities match; Praxis version and build provenance
+  remain attribution fields so candidate builds may differ. Unknown token/cost
+  evidence produces null deltas, while the gate requires no pass-rate or
+  safety-rate regression, every candidate verifier to be satisfied, and every
+  candidate `high` or `release` task to pass, and rejects incomplete safety
+  evidence.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,
@@ -401,9 +407,10 @@ normal/low-capability full-frame p95 budgets of `<16.7/<33 ms`.
 `npm run test:coverage` measures all production code under `src/**` with V8 and
 enforces global floors of 79% statements, 70% branches, 85% functions, and 81% lines,
 and rejects any production runtime module with zero covered statements (while allowing
-type-only modules). `npm run test:fixtures` executes the 71-behavior native contract; 63 behaviors
-are qualified and 8 are explicitly excluded. `npm run verify:fixture-contracts`
-performs the structural check and is part of `npm run check`.
+type-only modules). `npm run test:fixtures` executes the 74-behavior native contract; 66 behaviors
+are qualified and 8 are explicitly excluded. Schema-v2 risk tiers and executable evidence dimensions
+are enforced fail-closed. `npm run verify:fixture-contracts` performs the structural check and is part
+of `npm run check`.
 `npm run test:core-completion` is retained as a compatibility alias for
 `npm run test:fixtures`.
 

package/dist/build-identity.json

@@ -0,0 +1 @@
+{"schema_version":"1.0","source_revision":"git:8ed6222ee3a981ea0a00bb484ad43e13e278ab5d","source_dirty":false,"artifact_sha256":"sha256:8eb26913c75e9288f8387f5af1b5aefe847942e4c1d35598130cdac53d758524"}

package/dist/cli-runtime.js

@@ -51,7 +51,7 @@ import { FallbackModelProvider } from './providers/fallback-provider.js';
 import { ProviderCredentialVault } from './persistence/provider-credential-vault.js';
 import { parseContextEnvironment, parseProviderEnvironment, } from './providers/environment.js';
 import { ProviderAuthenticationError, resolveProviderCredential, } from './providers/provider-auth.js';
-import { resolveProviderContextWindowTokens, resolveProviderRegistry, } from './providers/provider-registry.js';
+import { resolveProviderContextWindowTokens, resolveProviderRegistry, resolveProviderRuntimeTarget, } from './providers/provider-registry.js';
 import { ProviderSettingsError, resolveProviderTarget, } from './providers/provider-settings.js';
 import { ModelPricingRegistry, usageCostUsd } from './core/usage.js';
 import { LocalToolRegistry } from './tools/local-tools.js';
@@ -73,6 +73,7 @@ import { addClaudeMarketplace, disableAllNativePlugins, installClaudeMarketplace
 import { executeClaudePluginEvalCommand, PLUGIN_EVAL_HELP, } from './plugins/claude-plugin-eval.js';
 import { executeProjectEvalCommand, PROJECT_EVAL_HELP, } from './evals/project-eval.js';
 import { PROJECT_EVAL_COMPARE_HELP } from './evals/project-eval-comparison.js';
+import { loadPraxisBuildIdentity } from './platform/praxis-build-identity.js';
 import { CLAUDE_PLUGIN_PRUNE_HELP, CLAUDE_PLUGIN_TAG_HELP, executeClaudePluginPrune, planClaudePluginPrune, tagClaudePlugin, } from './plugins/claude-plugin-maintenance.js';
 import { formatDoctorReport, runDoctor } from './maintenance/doctor.js';
 import { runSelfUpdate, } from './maintenance/self-update.js';
@@ -2358,6 +2359,27 @@ const defaultPluginEvalRuntimeFactory = {
         };
     },
 };
+const defaultProjectEvalRuntimeFactory = {
+    create: (options) => defaultPluginEvalRuntimeFactory.create(options),
+    identify: async (options) => {
+        const environment = process.env;
+        const target = await resolveProviderRuntimeTarget({
+            configRoot: options.configRoot,
+            cwd: options.cwd,
+            environment,
+            ...(options.model === undefined ? {} : { model: options.model }),
+            includeSettings: true,
+            includeProjectSettings: false,
+        });
+        return {
+            providerId: target.providerId,
+            profileId: target.profileId,
+            protocol: target.protocol,
+            endpoint: target.baseUrl,
+            modelId: target.modelId,
+        };
+    },
+};
 const defaultPluginEvalJudge = {
     vote: async ({ criteria, focus, baseline, model, signal }) => {
         const environment = process.env;
@@ -2600,7 +2622,8 @@ export function createDefaultDependencies(entrypoint = fileURLToPath(import.meta
             judge: defaultPluginEvalJudge,
         },
         projectEval: {
-            runtimeFactory: defaultPluginEvalRuntimeFactory,
+            runtimeFactory: defaultProjectEvalRuntimeFactory,
+            loadBuildIdentity: () => loadPraxisBuildIdentity(),
             version: VERSION,
             configRoot: resolveDataPlaneRoot(),
         },

package/dist/evals/eval-contract.d.ts

@@ -27,6 +27,17 @@ export interface EvalRuntimeFactoryOptions {
 export interface EvalRuntimeFactory {
     create(options: EvalRuntimeFactoryOptions): Promise<EvalRuntime>;
 }
+export type EvalRuntimeFactoryIdentityOptions = Omit<EvalRuntimeFactoryOptions, 'eventSink'>;
+export interface EvalRuntimeIdentityDescriptor {
+    providerId: string;
+    profileId: string;
+    protocol: string;
+    endpoint: string;
+    modelId: string;
+}
+export interface IdentifiedEvalRuntimeFactory extends EvalRuntimeFactory {
+    identify(options: EvalRuntimeFactoryIdentityOptions): Promise<EvalRuntimeIdentityDescriptor>;
+}
 export interface EvalTraceEvent {
     type: string;
     tool?: string;

package/dist/evals/project-eval-comparison.d.ts

@@ -1,4 +1,5 @@
 import type { ProjectEvalAggregate } from './project-eval.js';
+import { type ProjectEvalRisk } from './project-eval-schema.js';
 export declare const PROJECT_EVAL_COMPARE_HELP = "Usage: praxis eval compare [options]\n\nCompare two completed project evaluation aggregate artifacts.\n\nOptions:\n  --baseline <aggregate-result.json>   Baseline aggregate artifact\n  --baseline-name <name>               Name shown for the baseline\n  --candidate <aggregate-result.json>  Candidate aggregate artifact\n  --candidate-name <name>              Name shown for the candidate\n  --output-dir <dir>                   Write comparison-result.json here\n  --json                               Print exactly one comparison JSON value\n  -h, --help                           Display help";
 export interface ProjectEvalCompareOptions {
     baseline?: string;
@@ -20,18 +21,20 @@ export interface ProjectEvalComparisonMetric<T = number | null> {
     delta: T;
 }
 export interface ProjectEvalComparisonResult {
-    schema_version: '1.0';
+    schema_version: '1.2';
     baseline: {
         name: string;
         source_path: string;
         version: string;
         model: string | null;
+        identity_sha256: `sha256:${string}`;
     };
     candidate: {
         name: string;
         source_path: string;
         version: string;
         model: string | null;
+        identity_sha256: `sha256:${string}`;
     };
     comparable_run_count: number;
     passed: boolean;
@@ -43,6 +46,8 @@ export interface ProjectEvalComparisonResult {
     }[];
     metrics: {
         pass_rate: ProjectEvalComparisonMetric<number>;
+        verification_pass_rate: ProjectEvalComparisonMetric<number | null>;
+        task_risk_pass_rate: Record<ProjectEvalRisk, ProjectEvalComparisonMetric<number | null>>;
         safety_pass_rate: ProjectEvalComparisonMetric<number | null>;
         average_turns: ProjectEvalComparisonMetric<number>;
         input_tokens: ProjectEvalComparisonMetric<number | null>;

package/dist/evals/project-eval-comparison.js

@@ -1,6 +1,9 @@
 import { mkdir, lstat, readFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { writeFileAtomically } from '../platform/atomic-write.js';
+import { assertProjectEvalIdentitiesComparable, validateProjectEvalAggregateIdentity, validateProjectEvalIdentity, } from './project-eval-identity.js';
+import { PROJECT_EVAL_SAFETY_CHECKS } from './project-eval-runner.js';
+import { PROJECT_EVAL_MAX_ITEMS, PROJECT_EVAL_RISKS, } from './project-eval-schema.js';
 const MAX_AGGREGATE_BYTES = 8 * 1024 * 1024;
 const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
 export const PROJECT_EVAL_COMPARE_HELP = `Usage: praxis eval compare [options]
@@ -106,12 +109,6 @@ function objectField(value, path) {
         fail(path, 'expected an object');
     return value;
 }
-function optionalGroup(value, keys, path) {
-    const present = keys.filter((key) => value[key] !== undefined);
-    if (present.length !== 0 && present.length !== keys.length)
-        fail(path, `fields must be all present or all absent: ${keys.join(', ')}`);
-    return present.length === keys.length;
-}
 function usageField(value, path) {
     if (value === null)
         return null;
@@ -130,17 +127,117 @@ function usageField(value, path) {
         fail(path, 'inputTokens and outputTokens are required');
     return usage;
 }
+const FIXED_CHECKS = [
+    'runtime',
+    'termination',
+    ...PROJECT_EVAL_SAFETY_CHECKS,
+    'expected-paths',
+];
+const MAX_COMPACT_CHECKS = FIXED_CHECKS.length + 2 * PROJECT_EVAL_MAX_ITEMS;
+function checkEvidence(value, path) {
+    if (!Array.isArray(value) || value.length > MAX_COMPACT_CHECKS)
+        fail(path, 'expected a bounded array');
+    const names = new Set();
+    const checks = value.map((item, index) => {
+        const check = objectField(item, `${path}[${index}]`);
+        const unknown = Object.keys(check).find((key) => !['name', 'passed'].includes(key));
+        if (unknown)
+            fail(`${path}[${index}]`, `unknown field: ${unknown}`);
+        const name = stringField(check.name, `${path}[${index}].name`);
+        boolField(check.passed, `${path}[${index}].passed`);
+        if (names.has(name))
+            fail(path, 'check names must be unique');
+        names.add(name);
+        const fixed = FIXED_CHECKS.includes(name);
+        const dynamic = name === 'graders' ||
+            /^verifier:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(name) ||
+            /^grader:[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u.test(name);
+        if (!fixed && !dynamic)
+            fail(`${path}[${index}].name`, 'unsupported check');
+        return { name, passed: check.passed };
+    });
+    for (const name of FIXED_CHECKS)
+        if (!names.has(name))
+            fail(path, `missing required check: ${name}`);
+    return checks;
+}
+function verificationEvidence(value, path) {
+    const evidence = objectField(value, path);
+    const unknownEvidence = Object.keys(evidence).find((key) => !['outcomes', 'satisfied'].includes(key));
+    if (unknownEvidence)
+        fail(path, `unknown field: ${unknownEvidence}`);
+    const raw = evidence.outcomes;
+    if (!Array.isArray(raw) || raw.length > 256)
+        fail(`${path}.outcomes`, 'expected a bounded array');
+    const names = new Set();
+    const outcomes = raw.map((item, index) => {
+        const outcome = objectField(item, `${path}.outcomes[${index}]`);
+        const unknown = Object.keys(outcome).find((key) => !['name', 'required', 'expect', 'status'].includes(key));
+        if (unknown)
+            fail(`${path}.outcomes[${index}]`, `unknown field: ${unknown}`);
+        const name = stringField(outcome.name, `${path}.outcomes[${index}].name`);
+        if (!IDENTIFIER.test(name))
+            fail(`${path}.outcomes[${index}].name`, 'unsafe verifier name');
+        if (names.has(name))
+            fail(`${path}.outcomes`, 'verifier names must be unique');
+        names.add(name);
+        if (outcome.required !== true)
+            fail(`${path}.outcomes[${index}].required`, 'must be true');
+        if (outcome.expect !== 'pass')
+            fail(`${path}.outcomes[${index}].expect`, 'must be pass');
+        if (outcome.status !== 'passed' &&
+            outcome.status !== 'failed' &&
+            outcome.status !== 'not_run')
+            fail(`${path}.outcomes[${index}].status`, 'invalid verifier status');
+        return {
+            name,
+            required: true,
+            expect: 'pass',
+            status: outcome.status,
+        };
+    });
+    boolField(evidence.satisfied, `${path}.satisfied`);
+    const satisfied = outcomes.every((outcome) => outcome.status === 'passed');
+    if (evidence.satisfied !== satisfied)
+        fail(`${path}.satisfied`, 'does not match verifier statuses');
+    return { outcomes, satisfied };
+}
 function validateRun(value, index) {
     const path = `runs[${index}]`;
     const run = objectField(value, path);
+    if (run.schema_version !== '1.2')
+        fail(`${path}.schema_version`, 'must be "1.2"');
     stringField(run.case, `${path}.case`);
     if (!IDENTIFIER.test(run.case))
         fail(`${path}.case`, 'unsafe case name');
     const runNumber = numberField(run.run, `${path}.run`, true);
     if (runNumber < 1)
         fail(`${path}.run`, 'must be positive');
-    stringField(run.model, `${path}.model`, true);
+    const model = stringField(run.model, `${path}.model`);
+    const identity = validateProjectEvalIdentity(run.identity);
+    if (model !== identity.model_id)
+        fail(`${path}.model`, 'does not match identity.model_id');
+    if (!PROJECT_EVAL_RISKS.includes(run.risk))
+        fail(`${path}.risk`, 'invalid risk');
+    const checks = checkEvidence(run.checks, `${path}.checks`);
+    const verification = verificationEvidence(run.verification, `${path}.verification`);
+    const verifierChecks = new Map(checks
+        .filter(({ name }) => name.startsWith('verifier:'))
+        .map((check) => [check.name.slice('verifier:'.length), check.passed]));
+    if (verifierChecks.size !== verification.outcomes.length)
+        fail(`${path}.checks`, 'verifier checks do not match outcomes');
+    for (const outcome of verification.outcomes) {
+        const checkPassed = verifierChecks.get(outcome.name);
+        if (checkPassed === undefined)
+            fail(`${path}.verification`, 'missing verifier check');
+        if (checkPassed !== (outcome.status === 'passed'))
+            fail(`${path}.verification`, 'verifier check does not match status');
+    }
+    const recomputedPassed = checks.every((check) => check.passed);
+    const safetyPassed = PROJECT_EVAL_SAFETY_CHECKS.every((name) => checks.find((check) => check.name === name)?.passed === true);
     boolField(run.passed, `${path}.passed`);
+    if (run.passed !== recomputedPassed)
+        fail(`${path}.passed`, 'does not match compact checks');
     if (run.score !== 0 && run.score !== 1)
         fail(`${path}.score`, 'must be 0 or 1');
     if (run.score !== (run.passed ? 1 : 0))
@@ -158,16 +255,24 @@ function validateRun(value, index) {
         fail(`${path}.termination`, 'invalid termination');
     stringField(run.error, `${path}.error`, true);
     stringField(run.artifact_dir, `${path}.artifact_dir`);
-    const evidenceKnown = optionalGroup(run, ['safety_passed', 'permission_decisions', 'tool_errors', 'retries'], path);
-    if (evidenceKnown) {
-        boolField(run.safety_passed, `${path}.safety_passed`);
-        const permissions = objectField(run.permission_decisions, `${path}.permission_decisions`);
-        for (const key of ['allow', 'ask', 'deny'])
-            numberField(permissions[key], `${path}.permission_decisions.${key}`, true);
-        numberField(run.tool_errors, `${path}.tool_errors`, true);
-        numberField(run.retries, `${path}.retries`, true);
-    }
-    return run;
+    boolField(run.safety_passed, `${path}.safety_passed`);
+    if (run.safety_passed !== safetyPassed)
+        fail(`${path}.safety_passed`, 'does not match safety checks');
+    const permissions = objectField(run.permission_decisions, `${path}.permission_decisions`);
+    for (const key of ['allow', 'ask', 'deny'])
+        numberField(permissions[key], `${path}.permission_decisions.${key}`, true);
+    numberField(run.tool_errors, `${path}.tool_errors`, true);
+    numberField(run.retries, `${path}.retries`, true);
+    const terminationCheck = checks.find((check) => check.name === 'termination');
+    if (terminationCheck?.passed !== (run.termination === null))
+        fail(`${path}.checks`, 'termination check does not match termination');
+    return {
+        ...run,
+        model,
+        identity,
+        checks,
+        verification,
+    };
 }
 export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cwd()) {
     const sourcePath = resolve(callerCwd, inputPath);
@@ -190,8 +295,8 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     }
     const aggregate = objectField(value, 'root');
     const data = aggregate;
-    if (aggregate.schema_version !== '1.0')
-        fail('schema_version', 'must be "1.0"');
+    if (aggregate.schema_version !== '1.2')
+        fail('schema_version', 'must be "1.2"; legacy "1.1" aggregates are unsupported');
     stringField(aggregate.version, 'version');
     stringField(aggregate.start, 'start');
     numberField(aggregate.duration_ms, 'duration_ms');
@@ -230,6 +335,14 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     if (!Array.isArray(aggregate.runs) || aggregate.runs.length > 100000)
         fail('runs', 'expected a bounded array');
     const runs = aggregate.runs.map(validateRun);
+    for (const [index, run] of runs.entries())
+        if (run.identity.runtime.praxis_version !== aggregate.version)
+            fail(`runs[${index}].identity.runtime.praxis_version`, 'must match aggregate version');
+    const expectedAggregateModel = runs.length > 0 && runs.every((run) => run.model === runs[0]?.model)
+        ? (runs[0]?.model ?? null)
+        : null;
+    if (aggregate.model !== expectedAggregateModel)
+        fail('model', 'does not match completed run identity models');
     if (data.run_count !== runs.length)
         fail('run_count', 'does not match runs length');
     if (data.completed_run_count > data.planned_run_count ||
@@ -285,6 +398,11 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
         keys.add(key);
         caseNames.add(run.case);
     }
+    validateProjectEvalAggregateIdentity(aggregate.identity_sha256, runs.map((run) => ({
+        case: run.case,
+        run: run.run,
+        identity_sha256: run.identity.identity_sha256,
+    })));
     if (!data.partial && caseNames.size !== data.case_count)
         fail('case_count', 'does not match completed run cases');
     if (data.interrupted && !data.partial)
@@ -294,64 +412,95 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     if (data.partial !==
         (data.interrupted || data.run_count < data.planned_run_count))
         fail('partial', 'does not match interrupted/completed run state');
-    const aggregateEvidenceKnown = optionalGroup(aggregate, [
-        'safety_passed',
-        'safety_failed',
-        'permission_decisions',
-        'tool_errors',
-        'retries',
-        'terminations',
-    ], 'root evidence');
-    const runEvidenceKnown = runs.every((run) => typeof run.safety_passed === 'boolean' &&
-        typeof run.tool_errors === 'number' &&
-        typeof run.retries === 'number' &&
-        run.permission_decisions !== undefined);
-    const runEvidenceAbsent = runs.every((run) => run.safety_passed === undefined &&
-        run.tool_errors === undefined &&
-        run.retries === undefined &&
-        run.permission_decisions === undefined);
-    if ((aggregateEvidenceKnown && !runEvidenceKnown) ||
-        (!aggregateEvidenceKnown && !runEvidenceAbsent))
-        fail('runs', 'run evidence must match aggregate evidence availability');
-    const safetyKnown = aggregateEvidenceKnown && runEvidenceKnown;
-    if (aggregateEvidenceKnown) {
-        numberField(aggregate.safety_passed, 'safety_passed', true);
-        numberField(aggregate.safety_failed, 'safety_failed', true);
-        const permissions = objectField(aggregate.permission_decisions, 'permission_decisions');
-        for (const key of ['allow', 'ask', 'deny'])
-            numberField(permissions[key], `permission_decisions.${key}`, true);
-        numberField(aggregate.tool_errors, 'tool_errors', true);
-        numberField(aggregate.retries, 'retries', true);
-        if (aggregate.safety_passed !==
-            runs.filter((run) => run.safety_passed).length ||
-            aggregate.safety_failed !==
-                runs.filter((run) => !run.safety_passed).length)
-            fail('safety_passed', 'does not match run safety evidence');
-        const expectedPermissions = {
-            allow: runs.reduce((total, run) => total + run.permission_decisions.allow, 0),
-            ask: runs.reduce((total, run) => total + run.permission_decisions.ask, 0),
-            deny: runs.reduce((total, run) => total + run.permission_decisions.deny, 0),
-        };
-        if (permissions.allow !== expectedPermissions.allow ||
-            permissions.ask !== expectedPermissions.ask ||
-            permissions.deny !== expectedPermissions.deny)
-            fail('permission_decisions', 'does not match run evidence');
-        if (aggregate.tool_errors !==
-            runs.reduce((total, run) => total + run.tool_errors, 0) ||
-            aggregate.retries !== runs.reduce((total, run) => total + run.retries, 0))
-            fail('tool_errors', 'does not match run evidence');
-        const terminations = objectField(aggregate.terminations, 'terminations');
-        for (const key of ['completed', 'timeout', 'interrupted'])
-            numberField(terminations[key], `terminations.${key}`, true);
-        const expectedTerminations = {
-            completed: runs.filter((run) => run.termination === null).length,
-            timeout: runs.filter((run) => run.termination === 'timeout').length,
-            interrupted: runs.filter((run) => run.termination === 'interrupted')
-                .length,
-        };
-        for (const key of Object.keys(expectedTerminations))
-            if (terminations[key] !== expectedTerminations[key])
-                fail(`terminations.${key}`, 'does not match runs');
+    const safetyKnown = true;
+    numberField(aggregate.safety_passed, 'safety_passed', true);
+    numberField(aggregate.safety_failed, 'safety_failed', true);
+    const permissions = objectField(aggregate.permission_decisions, 'permission_decisions');
+    for (const key of ['allow', 'ask', 'deny'])
+        numberField(permissions[key], `permission_decisions.${key}`, true);
+    numberField(aggregate.tool_errors, 'tool_errors', true);
+    numberField(aggregate.retries, 'retries', true);
+    if (aggregate.safety_passed !==
+        runs.filter((run) => run.safety_passed).length ||
+        aggregate.safety_failed !== runs.filter((run) => !run.safety_passed).length)
+        fail('safety_passed', 'does not match run safety evidence');
+    const expectedPermissions = {
+        allow: runs.reduce((total, run) => total + run.permission_decisions.allow, 0),
+        ask: runs.reduce((total, run) => total + run.permission_decisions.ask, 0),
+        deny: runs.reduce((total, run) => total + run.permission_decisions.deny, 0),
+    };
+    if (permissions.allow !== expectedPermissions.allow ||
+        permissions.ask !== expectedPermissions.ask ||
+        permissions.deny !== expectedPermissions.deny)
+        fail('permission_decisions', 'does not match run evidence');
+    if (aggregate.tool_errors !==
+        runs.reduce((total, run) => total + run.tool_errors, 0) ||
+        aggregate.retries !== runs.reduce((total, run) => total + run.retries, 0))
+        fail('tool_errors', 'does not match run evidence');
+    const terminations = objectField(aggregate.terminations, 'terminations');
+    for (const key of ['completed', 'timeout', 'interrupted'])
+        numberField(terminations[key], `terminations.${key}`, true);
+    const expectedTerminations = {
+        completed: runs.filter((run) => run.termination === null).length,
+        timeout: runs.filter((run) => run.termination === 'timeout').length,
+        interrupted: runs.filter((run) => run.termination === 'interrupted').length,
+    };
+    for (const key of Object.keys(expectedTerminations))
+        if (terminations[key] !== expectedTerminations[key])
+            fail(`terminations.${key}`, 'does not match runs');
+    const verificationTotals = objectField(aggregate.verification_totals, 'verification_totals');
+    const unknownVerificationTotal = Object.keys(verificationTotals).find((key) => ![
+        'declared',
+        'passed',
+        'failed',
+        'not_run',
+        'satisfied_runs',
+        'unsatisfied_runs',
+    ].includes(key));
+    if (unknownVerificationTotal)
+        fail('verification_totals', `unknown field: ${unknownVerificationTotal}`);
+    for (const key of [
+        'declared',
+        'passed',
+        'failed',
+        'not_run',
+        'satisfied_runs',
+        'unsatisfied_runs',
+    ])
+        numberField(verificationTotals[key], `verification_totals.${key}`, true);
+    const expectedVerificationTotals = {
+        declared: runs.reduce((total, run) => total + run.verification.outcomes.length, 0),
+        passed: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'passed')
+                .length, 0),
+        failed: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'failed')
+                .length, 0),
+        not_run: runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'not_run')
+                .length, 0),
+        satisfied_runs: runs.filter((run) => run.verification.satisfied).length,
+        unsatisfied_runs: runs.filter((run) => !run.verification.satisfied).length,
+    };
+    for (const key of Object.keys(expectedVerificationTotals))
+        if (verificationTotals[key] !== expectedVerificationTotals[key])
+            fail(`verification_totals.${key}`, 'does not match run verification evidence');
+    const riskTiers = objectField(aggregate.risk_tiers, 'risk_tiers');
+    const unknownRisk = Object.keys(riskTiers).find((key) => !PROJECT_EVAL_RISKS.includes(key));
+    if (unknownRisk)
+        fail('risk_tiers', `unknown field: ${unknownRisk}`);
+    for (const risk of PROJECT_EVAL_RISKS) {
+        const tier = objectField(riskTiers[risk], `risk_tiers.${risk}`);
+        const unknownTier = Object.keys(tier).find((key) => !['runs', 'passed', 'failed'].includes(key));
+        if (unknownTier)
+            fail(`risk_tiers.${risk}`, `unknown field: ${unknownTier}`);
+        for (const key of ['runs', 'passed', 'failed'])
+            numberField(tier[key], `risk_tiers.${risk}.${key}`, true);
+        const riskRuns = runs.filter((run) => run.risk === risk);
+        if (tier.runs !== riskRuns.length ||
+            tier.passed !== riskRuns.filter((run) => run.passed).length ||
+            tier.failed !== riskRuns.filter((run) => !run.passed).length)
+            fail(`risk_tiers.${risk}`, 'does not match run risk evidence');
     }
     return {
         aggregate: { ...aggregate, runs },
@@ -393,6 +542,13 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         throw new Error('Aggregates have different comparable run sets');
     if (leftRuns.length === 0)
         throw new Error('Comparison requires at least one completed run');
+    for (let index = 0; index < leftRuns.length; index += 1) {
+        const leftRun = leftRuns[index];
+        const rightRun = rightRuns[index];
+        if (!leftRun || !rightRun)
+            continue;
+        assertProjectEvalIdentitiesComparable(leftRun.identity, rightRun.identity, `Identity mismatch for (${leftRun.case}, ${leftRun.run})`);
+    }
     const regressions = rightRuns.flatMap((run, index) => leftRuns[index]?.passed && !run.passed
         ? [
             {
@@ -410,6 +566,21 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
     const rightSafety = safetyKnown
         ? rightRuns.filter((run) => run.safety_passed).length / rightRuns.length
         : null;
+    const verificationRate = (runs) => {
+        const declared = runs.reduce((total, run) => total + run.verification.outcomes.length, 0);
+        if (declared === 0)
+            return null;
+        const passed = runs.reduce((total, run) => total +
+            run.verification.outcomes.filter(({ status }) => status === 'passed')
+                .length, 0);
+        return passed / declared;
+    };
+    const taskRiskRate = (runs, risk) => {
+        const selected = runs.filter((run) => run.risk === risk);
+        return selected.length === 0
+            ? null
+            : selected.filter((run) => run.passed).length / selected.length;
+    };
     const terms = (runs, kind) => runs.filter((run) => kind === 'completed'
         ? run.termination === null
         : run.termination === kind).length;
@@ -418,27 +589,38 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         ? metric(leftRuns.reduce((n, r) => n + r.permission_decisions[name], 0), rightRuns.reduce((n, r) => n + r.permission_decisions[name], 0))
         : nullableMetric(null, null);
     const result = {
-        schema_version: '1.0',
+        schema_version: '1.2',
         baseline: {
             name: baselineName,
             source_path: baseline.sourcePath,
             version: left.version,
             model: left.model,
+            identity_sha256: left.identity_sha256,
         },
         candidate: {
             name: candidateName,
             source_path: candidate.sourcePath,
             version: right.version,
             model: right.model,
+            identity_sha256: right.identity_sha256,
         },
         comparable_run_count: leftRuns.length,
         passed: regressions.length === 0 &&
             right.pass_rate >= left.pass_rate &&
             safetyKnown &&
-            (rightSafety ?? 0) >= (leftSafety ?? 0),
+            (rightSafety ?? 0) >= (leftSafety ?? 0) &&
+            rightRuns.every((run) => run.verification.satisfied) &&
+            rightRuns
+                .filter((run) => run.risk === 'high' || run.risk === 'release')
+                .every((run) => run.passed),
         regressions,
         metrics: {
             pass_rate: metric(left.pass_rate, right.pass_rate),
+            verification_pass_rate: nullableMetric(verificationRate(leftRuns), verificationRate(rightRuns)),
+            task_risk_pass_rate: Object.fromEntries(PROJECT_EVAL_RISKS.map((risk) => [
+                risk,
+                nullableMetric(taskRiskRate(leftRuns, risk), taskRiskRate(rightRuns, risk)),
+            ])),
             safety_pass_rate: nullableMetric(leftSafety, rightSafety),
             average_turns: metric(avg(leftRuns.map((r) => r.turns)), avg(rightRuns.map((r) => r.turns))),
             input_tokens: token('input_tokens'),

package/dist/evals/project-eval-identity.d.ts

@@ -0,0 +1,54 @@
+import type { EvalRuntimeIdentityDescriptor } from './eval-contract.js';
+import type { ProjectEvalCase } from './project-eval-schema.js';
+import type { FileManifest } from './project-eval-workspace.js';
+import { type PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
+export declare const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION: "1.1";
+export type ProjectEvalIdentitySchemaVersion = typeof PROJECT_EVAL_IDENTITY_SCHEMA_VERSION;
+export type IdentityDigest = `sha256:${string}`;
+export interface ProjectEvalRuntimeIdentity {
+    engine: 'praxis';
+    praxis_version: string;
+    node_version: string;
+    platform: string;
+    architecture: string;
+    build: PraxisBuildIdentity;
+    runtime_sha256: IdentityDigest;
+}
+export interface ProjectEvalIdentity {
+    schema_version: ProjectEvalIdentitySchemaVersion;
+    provider_id: string;
+    profile_id: string;
+    protocol: string;
+    endpoint_sha256: IdentityDigest;
+    model_id: string;
+    configuration_sha256: IdentityDigest;
+    tools_sha256: IdentityDigest;
+    prompt_sha256: IdentityDigest;
+    corpus_sha256: IdentityDigest;
+    runtime: ProjectEvalRuntimeIdentity;
+    identity_sha256: IdentityDigest;
+}
+export interface CreateProjectEvalIdentityInput {
+    provider: EvalRuntimeIdentityDescriptor;
+    case: ProjectEvalCase;
+    sourceBefore: FileManifest;
+    effectiveTools: readonly string[];
+    runVerification: boolean;
+    praxisVersion: string;
+    nodeVersion?: string;
+    platform?: string;
+    architecture?: string;
+    buildIdentity: PraxisBuildIdentity;
+}
+export interface AggregateIdentityRun {
+    case: string;
+    run: number;
+    identity_sha256: IdentityDigest;
+}
+export declare function createProjectEvalIdentity(input: CreateProjectEvalIdentityInput): ProjectEvalIdentity;
+export declare function validateProjectEvalIdentity(value: unknown): ProjectEvalIdentity;
+export declare function computeProjectEvalAggregateIdentity(runs: readonly AggregateIdentityRun[]): IdentityDigest;
+export declare function validateProjectEvalAggregateIdentity(value: unknown, runs: readonly AggregateIdentityRun[]): IdentityDigest;
+export declare function firstProjectEvalIdentityMismatch(left: ProjectEvalIdentity, right: ProjectEvalIdentity): string | null;
+export declare function assertProjectEvalIdentitiesComparable(left: ProjectEvalIdentity, right: ProjectEvalIdentity, context?: string): void;
+//# sourceMappingURL=project-eval-identity.d.ts.map