praxis-agent 0.64.1 → 0.65.0

package/README.md

@@ -134,10 +134,12 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
 
 - **Outcome-driven evaluation** — `praxis eval <target>` runs contained cases
   in isolated workspaces, requires explicit verifier authorization, and writes
-  versioned artifacts locally; usage and cost remain explicitly available or
-  unknown. Separate runs can be compared with `praxis eval compare`; unknown
-  token/cost evidence produces null deltas, while the gate requires no pass-rate
-  or safety-rate regression and rejects incomplete safety evidence.
+  versioned artifacts with deterministic run identities locally; usage and cost
+  remain explicitly available or unknown. Separate runs can be compared with
+  `praxis eval compare` only when provider, model, configuration, tools, prompt,
+  corpus, and runtime-environment identities match. Unknown token/cost evidence
+  produces null deltas, while the gate requires no pass-rate or safety-rate
+  regression and rejects incomplete safety evidence.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,

package/dist/cli-runtime.js

@@ -51,7 +51,7 @@ import { FallbackModelProvider } from './providers/fallback-provider.js';
 import { ProviderCredentialVault } from './persistence/provider-credential-vault.js';
 import { parseContextEnvironment, parseProviderEnvironment, } from './providers/environment.js';
 import { ProviderAuthenticationError, resolveProviderCredential, } from './providers/provider-auth.js';
-import { resolveProviderContextWindowTokens, resolveProviderRegistry, } from './providers/provider-registry.js';
+import { resolveProviderContextWindowTokens, resolveProviderRegistry, resolveProviderRuntimeTarget, } from './providers/provider-registry.js';
 import { ProviderSettingsError, resolveProviderTarget, } from './providers/provider-settings.js';
 import { ModelPricingRegistry, usageCostUsd } from './core/usage.js';
 import { LocalToolRegistry } from './tools/local-tools.js';
@@ -2358,6 +2358,27 @@ const defaultPluginEvalRuntimeFactory = {
         };
     },
 };
+const defaultProjectEvalRuntimeFactory = {
+    create: (options) => defaultPluginEvalRuntimeFactory.create(options),
+    identify: async (options) => {
+        const environment = process.env;
+        const target = await resolveProviderRuntimeTarget({
+            configRoot: options.configRoot,
+            cwd: options.cwd,
+            environment,
+            ...(options.model === undefined ? {} : { model: options.model }),
+            includeSettings: true,
+            includeProjectSettings: false,
+        });
+        return {
+            providerId: target.providerId,
+            profileId: target.profileId,
+            protocol: target.protocol,
+            endpoint: target.baseUrl,
+            modelId: target.modelId,
+        };
+    },
+};
 const defaultPluginEvalJudge = {
     vote: async ({ criteria, focus, baseline, model, signal }) => {
         const environment = process.env;
@@ -2600,7 +2621,7 @@ export function createDefaultDependencies(entrypoint = fileURLToPath(import.meta
             judge: defaultPluginEvalJudge,
         },
         projectEval: {
-            runtimeFactory: defaultPluginEvalRuntimeFactory,
+            runtimeFactory: defaultProjectEvalRuntimeFactory,
             version: VERSION,
             configRoot: resolveDataPlaneRoot(),
         },

package/dist/evals/eval-contract.d.ts

@@ -27,6 +27,17 @@ export interface EvalRuntimeFactoryOptions {
 export interface EvalRuntimeFactory {
     create(options: EvalRuntimeFactoryOptions): Promise<EvalRuntime>;
 }
+export type EvalRuntimeFactoryIdentityOptions = Omit<EvalRuntimeFactoryOptions, 'eventSink'>;
+export interface EvalRuntimeIdentityDescriptor {
+    providerId: string;
+    profileId: string;
+    protocol: string;
+    endpoint: string;
+    modelId: string;
+}
+export interface IdentifiedEvalRuntimeFactory extends EvalRuntimeFactory {
+    identify(options: EvalRuntimeFactoryIdentityOptions): Promise<EvalRuntimeIdentityDescriptor>;
+}
 export interface EvalTraceEvent {
     type: string;
     tool?: string;

package/dist/evals/project-eval-comparison.d.ts

@@ -20,18 +20,20 @@ export interface ProjectEvalComparisonMetric<T = number | null> {
     delta: T;
 }
 export interface ProjectEvalComparisonResult {
-    schema_version: '1.0';
+    schema_version: '1.1';
     baseline: {
         name: string;
         source_path: string;
         version: string;
         model: string | null;
+        identity_sha256: `sha256:${string}`;
     };
     candidate: {
         name: string;
         source_path: string;
         version: string;
         model: string | null;
+        identity_sha256: `sha256:${string}`;
     };
     comparable_run_count: number;
     passed: boolean;

package/dist/evals/project-eval-comparison.js

@@ -1,6 +1,7 @@
 import { mkdir, lstat, readFile } from 'node:fs/promises';
 import { dirname, resolve } from 'node:path';
 import { writeFileAtomically } from '../platform/atomic-write.js';
+import { assertProjectEvalIdentitiesComparable, validateProjectEvalAggregateIdentity, validateProjectEvalIdentity, } from './project-eval-identity.js';
 const MAX_AGGREGATE_BYTES = 8 * 1024 * 1024;
 const IDENTIFIER = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
 export const PROJECT_EVAL_COMPARE_HELP = `Usage: praxis eval compare [options]
@@ -139,7 +140,10 @@ function validateRun(value, index) {
     const runNumber = numberField(run.run, `${path}.run`, true);
     if (runNumber < 1)
         fail(`${path}.run`, 'must be positive');
-    stringField(run.model, `${path}.model`, true);
+    const model = stringField(run.model, `${path}.model`);
+    const identity = validateProjectEvalIdentity(run.identity);
+    if (model !== identity.model_id)
+        fail(`${path}.model`, 'does not match identity.model_id');
     boolField(run.passed, `${path}.passed`);
     if (run.score !== 0 && run.score !== 1)
         fail(`${path}.score`, 'must be 0 or 1');
@@ -167,7 +171,7 @@ function validateRun(value, index) {
         numberField(run.tool_errors, `${path}.tool_errors`, true);
         numberField(run.retries, `${path}.retries`, true);
     }
-    return run;
+    return { ...run, model, identity };
 }
 export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cwd()) {
     const sourcePath = resolve(callerCwd, inputPath);
@@ -190,8 +194,8 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     }
     const aggregate = objectField(value, 'root');
     const data = aggregate;
-    if (aggregate.schema_version !== '1.0')
-        fail('schema_version', 'must be "1.0"');
+    if (aggregate.schema_version !== '1.1')
+        fail('schema_version', 'must be "1.1"; legacy "1.0" aggregates are unsupported');
     stringField(aggregate.version, 'version');
     stringField(aggregate.start, 'start');
     numberField(aggregate.duration_ms, 'duration_ms');
@@ -230,6 +234,14 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
     if (!Array.isArray(aggregate.runs) || aggregate.runs.length > 100000)
         fail('runs', 'expected a bounded array');
     const runs = aggregate.runs.map(validateRun);
+    for (const [index, run] of runs.entries())
+        if (run.identity.runtime.praxis_version !== aggregate.version)
+            fail(`runs[${index}].identity.runtime.praxis_version`, 'must match aggregate version');
+    const expectedAggregateModel = runs.length > 0 && runs.every((run) => run.model === runs[0]?.model)
+        ? (runs[0]?.model ?? null)
+        : null;
+    if (aggregate.model !== expectedAggregateModel)
+        fail('model', 'does not match completed run identity models');
     if (data.run_count !== runs.length)
         fail('run_count', 'does not match runs length');
     if (data.completed_run_count > data.planned_run_count ||
@@ -285,6 +297,11 @@ export async function loadProjectEvalAggregate(inputPath, callerCwd = process.cw
         keys.add(key);
         caseNames.add(run.case);
     }
+    validateProjectEvalAggregateIdentity(aggregate.identity_sha256, runs.map((run) => ({
+        case: run.case,
+        run: run.run,
+        identity_sha256: run.identity.identity_sha256,
+    })));
     if (!data.partial && caseNames.size !== data.case_count)
         fail('case_count', 'does not match completed run cases');
     if (data.interrupted && !data.partial)
@@ -393,6 +410,13 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         throw new Error('Aggregates have different comparable run sets');
     if (leftRuns.length === 0)
         throw new Error('Comparison requires at least one completed run');
+    for (let index = 0; index < leftRuns.length; index += 1) {
+        const leftRun = leftRuns[index];
+        const rightRun = rightRuns[index];
+        if (!leftRun || !rightRun)
+            continue;
+        assertProjectEvalIdentitiesComparable(leftRun.identity, rightRun.identity, `Identity mismatch for (${leftRun.case}, ${leftRun.run})`);
+    }
     const regressions = rightRuns.flatMap((run, index) => leftRuns[index]?.passed && !run.passed
         ? [
             {
@@ -418,18 +442,20 @@ export function compareProjectEvalAggregates(baseline, candidate, baselineName,
         ? metric(leftRuns.reduce((n, r) => n + r.permission_decisions[name], 0), rightRuns.reduce((n, r) => n + r.permission_decisions[name], 0))
         : nullableMetric(null, null);
     const result = {
-        schema_version: '1.0',
+        schema_version: '1.1',
         baseline: {
             name: baselineName,
             source_path: baseline.sourcePath,
             version: left.version,
             model: left.model,
+            identity_sha256: left.identity_sha256,
         },
         candidate: {
             name: candidateName,
             source_path: candidate.sourcePath,
             version: right.version,
             model: right.model,
+            identity_sha256: right.identity_sha256,
         },
         comparable_run_count: leftRuns.length,
         passed: regressions.length === 0 &&

package/dist/evals/project-eval-identity.d.ts

@@ -0,0 +1,51 @@
+import type { EvalRuntimeIdentityDescriptor } from './eval-contract.js';
+import type { ProjectEvalCase } from './project-eval-schema.js';
+import type { FileManifest } from './project-eval-workspace.js';
+export declare const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION: "1.0";
+export type ProjectEvalIdentitySchemaVersion = typeof PROJECT_EVAL_IDENTITY_SCHEMA_VERSION;
+export type IdentityDigest = `sha256:${string}`;
+export interface ProjectEvalRuntimeIdentity {
+    engine: 'praxis';
+    praxis_version: string;
+    node_version: string;
+    platform: string;
+    architecture: string;
+    runtime_sha256: IdentityDigest;
+}
+export interface ProjectEvalIdentity {
+    schema_version: ProjectEvalIdentitySchemaVersion;
+    provider_id: string;
+    profile_id: string;
+    protocol: string;
+    endpoint_sha256: IdentityDigest;
+    model_id: string;
+    configuration_sha256: IdentityDigest;
+    tools_sha256: IdentityDigest;
+    prompt_sha256: IdentityDigest;
+    corpus_sha256: IdentityDigest;
+    runtime: ProjectEvalRuntimeIdentity;
+    identity_sha256: IdentityDigest;
+}
+export interface CreateProjectEvalIdentityInput {
+    provider: EvalRuntimeIdentityDescriptor;
+    case: ProjectEvalCase;
+    sourceBefore: FileManifest;
+    effectiveTools: readonly string[];
+    runVerification: boolean;
+    praxisVersion: string;
+    nodeVersion?: string;
+    platform?: string;
+    architecture?: string;
+}
+export interface AggregateIdentityRun {
+    case: string;
+    run: number;
+    identity_sha256: IdentityDigest;
+}
+export declare function createProjectEvalIdentity(input: CreateProjectEvalIdentityInput): ProjectEvalIdentity;
+export declare function validateProjectEvalIdentity(value: unknown): ProjectEvalIdentity;
+export declare function computeProjectEvalAggregateIdentity(runs: readonly AggregateIdentityRun[]): IdentityDigest;
+export declare function validateProjectEvalAggregateIdentity(value: unknown, runs: readonly AggregateIdentityRun[]): IdentityDigest;
+export declare function firstProjectEvalIdentityMismatch(left: ProjectEvalIdentity, right: ProjectEvalIdentity): string | null;
+export declare function assertProjectEvalIdentitiesComparable(left: ProjectEvalIdentity, right: ProjectEvalIdentity, context?: string): void;
+//# sourceMappingURL=project-eval-identity.d.ts.map

package/dist/evals/project-eval-identity.js

@@ -0,0 +1,341 @@
+import { createHash } from 'node:crypto';
+import { posix, win32 } from 'node:path';
+import { redactSensitiveValue, sensitiveEnvironmentValues, } from '../platform/sensitive-data.js';
+export const PROJECT_EVAL_IDENTITY_SCHEMA_VERSION = '1.0';
+const DIGEST = /^sha256:[0-9a-f]{64}$/u;
+const MAX_DEPTH = 32;
+const MAX_NODES = 500_000;
+const MAX_STRING = 64 * 1024;
+function fail(message) {
+    throw new Error(`Invalid Project Eval identity: ${message}`);
+}
+function codeUnitCompare(left, right) {
+    return left < right ? -1 : left > right ? 1 : 0;
+}
+function assertSerializable(value, path = '$', depth = 0, state = { nodes: 0 }, ancestry = new WeakSet()) {
+    state.nodes += 1;
+    if (state.nodes > MAX_NODES)
+        fail(`${path} exceeds object node limit`);
+    if (depth > MAX_DEPTH)
+        fail(`${path} exceeds object depth limit`);
+    if (value === null ||
+        typeof value === 'boolean' ||
+        typeof value === 'number') {
+        if (typeof value === 'number' && !Number.isFinite(value))
+            fail(`${path} contains a non-finite number`);
+        return;
+    }
+    if (typeof value === 'string') {
+        if (value.length > MAX_STRING)
+            fail(`${path} contains an oversized string`);
+        return;
+    }
+    if (typeof value === 'bigint' ||
+        typeof value === 'function' ||
+        typeof value === 'symbol' ||
+        value === undefined)
+        fail(`${path} contains an unsupported value`);
+    if (Array.isArray(value)) {
+        if (ancestry.has(value))
+            fail(`${path} contains a circular reference`);
+        ancestry.add(value);
+        try {
+            for (let index = 0; index < value.length; index += 1)
+                assertSerializable(value[index], `${path}[${index}]`, depth + 1, state, ancestry);
+        }
+        finally {
+            ancestry.delete(value);
+        }
+        return;
+    }
+    if (typeof value === 'object') {
+        const prototype = Object.getPrototypeOf(value);
+        if (prototype !== Object.prototype && prototype !== null)
+            fail(`${path} contains an unsupported object`);
+        if (ancestry.has(value))
+            fail(`${path} contains a circular reference`);
+        ancestry.add(value);
+        try {
+            for (const [key, child] of Object.entries(value)) {
+                if (key.length > 512)
+                    fail(`${path} contains an oversized key`);
+                assertSerializable(child, `${path}.${key}`, depth + 1, state, ancestry);
+            }
+        }
+        finally {
+            ancestry.delete(value);
+        }
+        return;
+    }
+    fail(`${path} contains an unsupported value`);
+}
+function canonical(value) {
+    assertSerializable(value);
+    const ancestry = new WeakSet();
+    const serialize = (item) => {
+        if (item === null || typeof item !== 'object')
+            return JSON.stringify(item);
+        if (ancestry.has(item))
+            fail('contains a circular reference');
+        ancestry.add(item);
+        try {
+            if (Array.isArray(item))
+                return `[${item.map((child) => serialize(child)).join(',')}]`;
+            return `{${Object.entries(item)
+                .sort(([left], [right]) => codeUnitCompare(left, right))
+                .map(([key, child]) => `${JSON.stringify(key)}:${serialize(child)}`)
+                .join(',')}}`;
+        }
+        finally {
+            ancestry.delete(item);
+        }
+    };
+    return serialize(value);
+}
+function digest(value) {
+    return `sha256:${createHash('sha256').update(canonical(value), 'utf8').digest('hex')}`;
+}
+function string(value, path) {
+    if (typeof value !== 'string' ||
+        value.length === 0 ||
+        value.length > MAX_STRING)
+        fail(`${path} must be a bounded non-empty string`);
+    return value;
+}
+function digestField(value, path) {
+    if (typeof value !== 'string' || !DIGEST.test(value))
+        fail(`${path} must be a sha256 digest`);
+    return value;
+}
+function positiveInteger(value, path) {
+    if (!Number.isSafeInteger(value) || value < 1)
+        fail(`${path} must be a positive integer`);
+    return value;
+}
+function configurationSource(input) {
+    const c = input.case;
+    const sensitiveValues = sensitiveEnvironmentValues(c.execution.env);
+    const graders = c.graders.map((grader) => grader.type === 'tool_used' && !Number.isFinite(grader.max)
+        ? { ...grader, max: 'unbounded' }
+        : grader);
+    const configuration = {
+        case: {
+            name: c.name,
+            schema_version: c.schemaVersion,
+        },
+        execution: {
+            max_turns: c.execution.maxTurns,
+            timeout_seconds: c.execution.timeoutSeconds,
+            env: c.execution.env,
+        },
+        verification: {
+            enabled: input.runVerification,
+            definitions: c.verification,
+        },
+        graders,
+        expect: {
+            allowed_changed_paths: c.expect.allowedChangedPaths,
+            expected_changed_paths: c.expect.expectedChangedPaths,
+            forbidden_changed_paths: c.expect.forbiddenChangedPaths,
+        },
+    };
+    return normalizeConfiguration(redactSensitiveValue(configuration, sensitiveValues));
+}
+function normalizeConfiguration(value) {
+    if (typeof value === 'string')
+        return posix.isAbsolute(value) || win32.isAbsolute(value)
+            ? '[ABSOLUTE_PATH]'
+            : value;
+    if (Array.isArray(value))
+        return value.map(normalizeConfiguration);
+    if (value && typeof value === 'object')
+        return Object.fromEntries(Object.entries(value).map(([key, child]) => [
+            key,
+            normalizeConfiguration(child),
+        ]));
+    return value;
+}
+function corpusSource(sourceBefore) {
+    return {
+        files: Object.fromEntries(Object.entries(sourceBefore.files).map(([path, file]) => [
+            path,
+            {
+                hash: file.hash,
+                size: file.size,
+                mode: file.mode,
+            },
+        ])),
+        total_bytes: sourceBefore.totalBytes,
+    };
+}
+function freeze(value) {
+    if (value && typeof value === 'object') {
+        Object.freeze(value);
+        for (const child of Object.values(value))
+            freeze(child);
+    }
+    return value;
+}
+export function createProjectEvalIdentity(input) {
+    const provider = input.provider;
+    const runtimeBase = {
+        engine: 'praxis',
+        praxis_version: string(input.praxisVersion, 'praxis_version'),
+        node_version: string(input.nodeVersion ?? process.version, 'node_version'),
+        platform: string(input.platform ?? process.platform, 'platform'),
+        architecture: string(input.architecture ?? process.arch, 'architecture'),
+    };
+    const identityWithoutDigests = {
+        schema_version: PROJECT_EVAL_IDENTITY_SCHEMA_VERSION,
+        provider_id: string(provider.providerId, 'provider_id'),
+        profile_id: string(provider.profileId, 'profile_id'),
+        protocol: string(provider.protocol, 'protocol'),
+        endpoint_sha256: digest(string(provider.endpoint, 'endpoint')),
+        model_id: string(provider.modelId, 'model_id'),
+        configuration_sha256: digest(configurationSource(input)),
+        tools_sha256: digest(input.effectiveTools),
+        prompt_sha256: digest({
+            prompt: input.case.execution.prompt,
+            append_system_prompt: input.case.execution.appendSystemPrompt ?? null,
+        }),
+        corpus_sha256: digest(corpusSource(input.sourceBefore)),
+        runtime: {
+            ...runtimeBase,
+            runtime_sha256: digest(runtimeBase),
+        },
+    };
+    return freeze({
+        ...identityWithoutDigests,
+        identity_sha256: digest(identityWithoutDigests),
+    });
+}
+function validateRuntime(value, path) {
+    const runtime = value;
+    if (!runtime || typeof runtime !== 'object' || Array.isArray(runtime))
+        fail(`${path} must be an object`);
+    const runtimeKeys = [
+        'engine',
+        'praxis_version',
+        'node_version',
+        'platform',
+        'architecture',
+        'runtime_sha256',
+    ];
+    const runtimeUnknown = Object.keys(runtime).find((key) => !runtimeKeys.includes(key));
+    if (runtimeUnknown)
+        fail(`${path}.${runtimeUnknown} is not supported`);
+    if (runtime.engine !== 'praxis')
+        fail(`${path}.engine must be "praxis"`);
+    const validated = {
+        engine: 'praxis',
+        praxis_version: string(runtime.praxis_version, `${path}.praxis_version`),
+        node_version: string(runtime.node_version, `${path}.node_version`),
+        platform: string(runtime.platform, `${path}.platform`),
+        architecture: string(runtime.architecture, `${path}.architecture`),
+        runtime_sha256: digestField(runtime.runtime_sha256, `${path}.runtime_sha256`),
+    };
+    if (validated.runtime_sha256 !==
+        digest({
+            engine: validated.engine,
+            praxis_version: validated.praxis_version,
+            node_version: validated.node_version,
+            platform: validated.platform,
+            architecture: validated.architecture,
+        }))
+        fail(`${path}.runtime_sha256 does not match runtime fields`);
+    return validated;
+}
+export function validateProjectEvalIdentity(value) {
+    if (!value || typeof value !== 'object' || Array.isArray(value))
+        fail('must be an object');
+    const source = value;
+    const keys = [
+        'schema_version',
+        'provider_id',
+        'profile_id',
+        'protocol',
+        'endpoint_sha256',
+        'model_id',
+        'configuration_sha256',
+        'tools_sha256',
+        'prompt_sha256',
+        'corpus_sha256',
+        'runtime',
+        'identity_sha256',
+    ];
+    const unknown = Object.keys(source).find((key) => !keys.includes(key));
+    if (unknown)
+        fail(`${unknown} is not supported`);
+    if (source.schema_version !== PROJECT_EVAL_IDENTITY_SCHEMA_VERSION)
+        fail('schema_version must be "1.0"');
+    const validated = {
+        schema_version: PROJECT_EVAL_IDENTITY_SCHEMA_VERSION,
+        provider_id: string(source.provider_id, 'provider_id'),
+        profile_id: string(source.profile_id, 'profile_id'),
+        protocol: string(source.protocol, 'protocol'),
+        endpoint_sha256: digestField(source.endpoint_sha256, 'endpoint_sha256'),
+        model_id: string(source.model_id, 'model_id'),
+        configuration_sha256: digestField(source.configuration_sha256, 'configuration_sha256'),
+        tools_sha256: digestField(source.tools_sha256, 'tools_sha256'),
+        prompt_sha256: digestField(source.prompt_sha256, 'prompt_sha256'),
+        corpus_sha256: digestField(source.corpus_sha256, 'corpus_sha256'),
+        runtime: validateRuntime(source.runtime, 'runtime'),
+        identity_sha256: digestField(source.identity_sha256, 'identity_sha256'),
+    };
+    const withoutIdentity = { ...validated };
+    delete withoutIdentity.identity_sha256;
+    if (validated.identity_sha256 !== digest(withoutIdentity))
+        fail('identity_sha256 does not match identity fields');
+    return freeze(validated);
+}
+export function computeProjectEvalAggregateIdentity(runs) {
+    const entries = runs
+        .map((run) => ({
+        case: string(run.case, 'runs.case'),
+        run: positiveInteger(run.run, 'runs.run'),
+        identity_sha256: digestField(run.identity_sha256, 'runs.identity_sha256'),
+    }))
+        .sort((left, right) => {
+        const caseOrder = codeUnitCompare(left.case, right.case);
+        return caseOrder === 0 ? left.run - right.run : caseOrder;
+    });
+    return digest(entries);
+}
+export function validateProjectEvalAggregateIdentity(value, runs) {
+    const declared = digestField(value, 'identity_sha256');
+    const expected = computeProjectEvalAggregateIdentity(runs);
+    if (declared !== expected)
+        fail('aggregate identity_sha256 does not match runs');
+    return declared;
+}
+const comparableIdentityDimensions = [
+    ['provider_id', 'provider_id'],
+    ['profile_id', 'profile_id'],
+    ['protocol', 'protocol'],
+    ['endpoint_sha256', 'endpoint_sha256'],
+    ['model_id', 'model_id'],
+    ['configuration_sha256', 'configuration_sha256'],
+    ['tools_sha256', 'tools_sha256'],
+    ['prompt_sha256', 'prompt_sha256'],
+    ['corpus_sha256', 'corpus_sha256'],
+    ['runtime.engine', 'runtime.engine'],
+    ['runtime.node_version', 'runtime.node_version'],
+    ['runtime.platform', 'platform'],
+    ['runtime.architecture', 'architecture'],
+];
+export function firstProjectEvalIdentityMismatch(left, right) {
+    for (const [field, label] of comparableIdentityDimensions) {
+        const read = (identity) => field.startsWith('runtime.')
+            ? identity.runtime[field.slice('runtime.'.length)]
+            : identity[field];
+        if (read(left) !== read(right))
+            return label;
+    }
+    return null;
+}
+export function assertProjectEvalIdentitiesComparable(left, right, context) {
+    const mismatch = firstProjectEvalIdentityMismatch(left, right);
+    if (mismatch)
+        throw new Error(`${context ? `${context}: ` : ''}non-comparable identity: ${mismatch} differs`);
+}
+//# sourceMappingURL=project-eval-identity.js.map

package/dist/evals/project-eval-runner.d.ts

@@ -1,5 +1,6 @@
 import type { ModelUsage } from '../core/runtime.js';
-import { type EvalGraderResult, type EvalRuntimeFactory } from './eval-contract.js';
+import { type EvalGraderResult, type IdentifiedEvalRuntimeFactory } from './eval-contract.js';
+import { type ProjectEvalIdentity } from './project-eval-identity.js';
 import type { ProjectEvalCase } from './project-eval-schema.js';
 export type ProjectEvalTermination = 'timeout' | 'interrupted' | null;
 export interface ProjectEvalVerificationResult {
@@ -17,11 +18,12 @@ export interface ProjectEvalVerificationResult {
     error: string | null;
 }
 export interface ProjectEvalRunResult {
-    schema_version: '1.0';
+    schema_version: '1.1';
     case: string;
     run: number;
     version: string;
-    model: string | null;
+    model: string;
+    identity: ProjectEvalIdentity;
     passed: boolean;
     score: 0 | 1;
     turns: number;
@@ -44,6 +46,7 @@ export interface ProjectEvalRunResult {
         trace: 'trace.jsonl';
         workspace_diff: 'workspace-diff.json';
         verification: 'verification.json';
+        identity: 'identity.json';
         result: 'result.json';
     };
     error: string | null;
@@ -52,7 +55,7 @@ export interface ProjectEvalRunResult {
 }
 interface ProjectEvalRunOptions {
     case: ProjectEvalCase;
-    factory: EvalRuntimeFactory;
+    factory: IdentifiedEvalRuntimeFactory;
     run: number;
     allowTools?: readonly string[];
     model?: string;

package/dist/evals/project-eval-runner.js

@@ -3,6 +3,7 @@ import { BoundedProcessRunner } from '../platform/bounded-process-runner.js';
 import { redactSensitiveValue, redactSensitiveText, sensitiveEnvironmentValues, } from '../platform/sensitive-data.js';
 import { gradeDeterministicEvalRun } from './eval-graders.js';
 import { normalizeEvalTraceEvent, resolveEvalAllowedTools, } from './eval-contract.js';
+import { createProjectEvalIdentity, } from './project-eval-identity.js';
 import { cleanupProjectEvalWorkspace, createProjectEvalWorkspace, diffProjectEvalWorkspace, } from './project-eval-workspace.js';
 import { minimatch } from 'minimatch';
 import { join } from 'node:path';
@@ -117,6 +118,38 @@ export async function runProjectEvalCase(options) {
     let lastMessage = '';
     let graderError = null;
     let graders = [];
+    const effectiveModel = options.model ?? options.case.execution.model;
+    const factoryOptions = {
+        dataPlane: 'native',
+        cwd: workspace.cwd,
+        configRoot: workspace.config,
+        home: workspace.home,
+        maxTurns: options.case.execution.maxTurns,
+        pluginDirectories: [],
+        allowedTools,
+        ...(effectiveModel ? { model: effectiveModel } : {}),
+        ...(options.case.execution.appendSystemPrompt
+            ? { appendSystemPrompt: options.case.execution.appendSystemPrompt }
+            : {}),
+        addDirs: [],
+        env: options.case.execution.env,
+    };
+    let identity;
+    try {
+        const descriptor = await options.factory.identify(factoryOptions);
+        identity = createProjectEvalIdentity({
+            provider: descriptor,
+            case: options.case,
+            sourceBefore: workspace.sourceBefore,
+            effectiveTools: allowedTools,
+            runVerification: options.runVerification ?? false,
+            praxisVersion: options.version,
+        });
+    }
+    catch (error) {
+        await cleanupProjectEvalWorkspace(workspace.root).catch(() => undefined);
+        throw error;
+    }
     const runtimeController = new AbortController();
     let deadlineReached = false;
     const forwardAbort = () => runtimeController.abort(options.signal?.reason);
@@ -129,21 +162,7 @@ export async function runProjectEvalCase(options) {
     }, options.case.execution.timeoutSeconds * 1000);
     try {
         runtime = await options.factory.create({
-            dataPlane: 'native',
-            cwd: workspace.cwd,
-            configRoot: workspace.config,
-            home: workspace.home,
-            maxTurns: options.case.execution.maxTurns,
-            pluginDirectories: [],
-            allowedTools,
-            ...((options.model ?? options.case.execution.model)
-                ? { model: options.model ?? options.case.execution.model }
-                : {}),
-            ...(options.case.execution.appendSystemPrompt
-                ? { appendSystemPrompt: options.case.execution.appendSystemPrompt }
-                : {}),
-            addDirs: [],
-            env: options.case.execution.env,
+            ...factoryOptions,
             eventSink: (event) => {
                 if (traceOverflow)
                     return;
@@ -332,6 +351,7 @@ export async function runProjectEvalCase(options) {
         await writeFileAtomically(join(runDirectory, 'trace.jsonl'), traceRecords.join('\n'));
         await writeFileAtomically(join(runDirectory, 'workspace-diff.json'), JSON.stringify({ schema_version: '1.0', ...workspaceDiff }, null, 2));
         await writeFileAtomically(join(runDirectory, 'verification.json'), JSON.stringify(verifications, null, 2));
+        await writeFileAtomically(join(runDirectory, 'identity.json'), JSON.stringify(identity, null, 2));
     }
     catch (error) {
         artifactError = errorText(error);
@@ -361,11 +381,12 @@ export async function runProjectEvalCase(options) {
         checks.find((item) => !item.passed)?.explanation ??
         null;
     const result = {
-        schema_version: '1.0',
+        schema_version: '1.1',
         case: options.case.name,
         run: options.run,
         version: options.version,
-        model: options.model ?? options.case.execution.model ?? null,
+        model: identity.model_id,
+        identity,
         passed,
         score: passed ? 1 : 0,
         turns,
@@ -381,6 +402,7 @@ export async function runProjectEvalCase(options) {
             trace: 'trace.jsonl',
             workspace_diff: 'workspace-diff.json',
             verification: 'verification.json',
+            identity: 'identity.json',
             result: 'result.json',
         },
         error: primaryError,

package/dist/evals/project-eval.d.ts

@@ -1,9 +1,10 @@
 import type { ModelUsage } from '../core/runtime.js';
-import type { EvalRuntimeFactory } from './eval-contract.js';
+import type { IdentifiedEvalRuntimeFactory } from './eval-contract.js';
 import { type ProjectEvalRunResult } from './project-eval-runner.js';
+import { type ProjectEvalIdentity } from './project-eval-identity.js';
 export declare const PROJECT_EVAL_HELP = "Usage: praxis eval [options] <target>\n\nRun deterministic project outcome evaluations in isolated workspaces.\n\nOptions:\n  --case <glob>          Filter case names\n  --tag <tag[,tag]>      Filter tags; repeatable\n  --runs <1..50>         Override run count\n  --model <model>        Override model\n  --allow-tools <rules>  Grant gated tools; comma-separated and repeatable\n  --run-verification     Enable verifier subprocesses\n  --output-dir <dir>     Write artifacts to this directory\n  --keep-temp            Preserve temporary workspaces\n  --json                 Print exactly one aggregate JSON value\n  --verbose              Print run progress to stderr\n  -h, --help             Display help\n\nUse praxis eval compare --help to compare two completed aggregate artifacts.";
 export interface ProjectEvalDependencies {
-    runtimeFactory: EvalRuntimeFactory;
+    runtimeFactory: IdentifiedEvalRuntimeFactory;
     version?: string;
     configRoot: string;
 }
@@ -31,7 +32,7 @@ export interface ProjectEvalUsageTotals {
 export interface ProjectEvalRunSummary {
     case: string;
     run: number;
-    model: string | null;
+    model: string;
     passed: boolean;
     score: 0 | 1;
     turns: number;
@@ -50,9 +51,10 @@ export interface ProjectEvalRunSummary {
     retries: number;
     error: string | null;
     artifact_dir: string;
+    identity: ProjectEvalIdentity;
 }
 export interface ProjectEvalAggregate {
-    schema_version: '1.0';
+    schema_version: '1.1';
     version: string;
     start: string;
     duration_ms: number;
@@ -89,6 +91,7 @@ export interface ProjectEvalAggregate {
     };
     partial: boolean;
     interrupted: boolean;
+    identity_sha256: `sha256:${string}`;
     runs: readonly ProjectEvalRunSummary[];
 }
 export declare function parseProjectEvalOptions(argv: readonly string[]): ProjectEvalOptions;

package/dist/evals/project-eval.js

@@ -1,6 +1,7 @@
 import { join, relative, resolve } from 'node:path';
 import { writeFileAtomically } from '../platform/atomic-write.js';
 import { runProjectEvalCase, } from './project-eval-runner.js';
+import { computeProjectEvalAggregateIdentity, } from './project-eval-identity.js';
 import { executeProjectEvalCompareCommand } from './project-eval-comparison.js';
 import { discoverProjectEvalCases } from './project-eval-schema.js';
 export const PROJECT_EVAL_HELP = `Usage: praxis eval [options] <target>
@@ -134,6 +135,7 @@ function runSummary(result, outputDirectory) {
         retries: result.retries,
         error: result.error,
         artifact_dir: relative(outputDirectory, join(outputDirectory, result.case, `run-${result.run}`)).replaceAll('\\', '/'),
+        identity: result.identity,
     };
 }
 export async function executeProjectEvalCommand(argv, io, dependencies, callerCwd = process.cwd(), signal) {
@@ -184,14 +186,18 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
     }
     const passed = results.filter((result) => result.passed).length;
     const knownCostResults = results.filter((result) => result.cost_known);
+    const summaries = results.map((result) => runSummary(result, outputDirectory));
     const aggregate = {
-        schema_version: '1.0',
+        schema_version: '1.1',
         version: dependencies.version ?? 'unknown',
         start: new Date(started).toISOString(),
         duration_ms: Date.now() - started,
         target,
         output_dir: outputDirectory,
-        model: options.model ?? null,
+        model: summaries.length > 0 &&
+            summaries.every((summary) => summary.model === summaries[0]?.model)
+            ? (summaries[0]?.model ?? null)
+            : null,
         case_count: cases.length,
         planned_run_count: plannedRunCount,
         completed_run_count: results.length,
@@ -226,7 +232,12 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
         },
         partial: interrupted || results.length < plannedRunCount,
         interrupted,
-        runs: results.map((result) => runSummary(result, outputDirectory)),
+        identity_sha256: computeProjectEvalAggregateIdentity(summaries.map((summary) => ({
+            case: summary.case,
+            run: summary.run,
+            identity_sha256: summary.identity.identity_sha256,
+        }))),
+        runs: summaries,
     };
     await writeFileAtomically(join(outputDirectory, 'aggregate-result.json'), JSON.stringify(aggregate, null, 2));
     if (options.json)

package/dist/providers/provider-registry.d.ts

@@ -1,6 +1,6 @@
 import type { ModelProvider, ModelThinkingConfig } from '../core/runtime.js';
 import { type CodexOAuthVault } from './codex-oauth.js';
-import { type ProviderProtocol, type ProviderTarget } from './provider-settings.js';
+import { type ResolveProviderTargetOptions, type ProviderProtocol, type ProviderTarget } from './provider-settings.js';
 import { type AnthropicModelAliasOverrides } from './anthropic-model-alias.js';
 import type { ProviderCredentialSourceMetadata, ProviderCredentialReader, ResolvedProviderCredential } from './provider-auth.js';
 import { parseProviderEnvironment, type ContextEnvironment } from './environment.js';
@@ -44,6 +44,7 @@ export interface ResolveProviderRegistryOptions {
     fetchImplementation?: typeof fetch;
 }
 export type ProviderRegistrySourceMetadata = ProviderCredentialSourceMetadata;
+export declare function resolveProviderRuntimeTarget(options: ResolveProviderTargetOptions): Promise<ProviderTarget>;
 export interface ProviderRegistry {
     readonly target: ProviderTarget;
     readonly credentialSource: ProviderRegistrySourceMetadata;

package/dist/providers/provider-registry.js

@@ -19,6 +19,24 @@ export class ProviderRegistryError extends Error {
         this.code = code;
     }
 }
+function effectiveProviderTarget(target, overrides) {
+    if (target.providerId !== 'anthropic' ||
+        target.protocol !== 'anthropic-messages')
+        return target;
+    return {
+        ...target,
+        modelId: resolveAnthropicModelAlias(target.modelId, overrides),
+    };
+}
+export async function resolveProviderRuntimeTarget(options) {
+    const environment = options.environment ?? process.env;
+    const target = await resolveProviderTarget({ ...options, environment });
+    const aliases = target.providerId === 'anthropic' &&
+        target.protocol === 'anthropic-messages'
+        ? anthropicModelAliasOverridesFromEnvironment(environment)
+        : undefined;
+    return effectiveProviderTarget(target, aliases);
+}
 export function resolveProviderContextWindowTokens(options) {
     if (options.protocol === 'anthropic-messages')
         return resolveAnthropicModelSpec(options.modelId, options.explicitContextWindowTokens).contextWindowTokens;
@@ -113,8 +131,10 @@ class NativeProviderRegistry {
                 : { fetchImplementation: options.fetchImplementation });
         }
     }
-    create(modelId = this.target.modelId) {
-        const target = this.resolveTarget({ ...this.target, modelId });
+    create(modelId) {
+        const target = modelId === undefined
+            ? this.target
+            : this.resolveTarget({ ...this.target, modelId });
         if (target.protocol === 'codex-subscription') {
             if (!this.codexManager)
                 throw new ProviderAuthenticationError('invalid_credential', 'Provider authentication failed: Codex subscription credentials are unavailable');
@@ -239,13 +259,7 @@ class NativeProviderRegistry {
         return override !== undefined && override.trim().length > 0;
     }
     resolveTarget(target) {
-        if (target.providerId !== 'anthropic' ||
-            target.protocol !== 'anthropic-messages')
-            return target;
-        return {
-            ...target,
-            modelId: resolveAnthropicModelAlias(target.modelId, this.options.anthropicModelAliasOverrides),
-        };
+        return effectiveProviderTarget(target, this.options.anthropicModelAliasOverrides);
     }
     withDeadline(provider) {
         const environment = this.options.providerEnvironment;

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "praxis-agent",
-  "version": "0.64.1",
+  "version": "0.65.0",
   "description": "Local-first, single-user general agent for the command line.",
   "license": "MIT",
   "author": "wuqisen",