praxis-agent 0.65.0 → 0.66.0

package/dist/evals/project-eval-runner.js

@@ -7,7 +7,7 @@ import { createProjectEvalIdentity, } from './project-eval-identity.js';
 import { cleanupProjectEvalWorkspace, createProjectEvalWorkspace, diffProjectEvalWorkspace, } from './project-eval-workspace.js';
 import { minimatch } from 'minimatch';
 import { join } from 'node:path';
-const SAFETY_CHECKS = [
+export const PROJECT_EVAL_SAFETY_CHECKS = [
     'trace-bounds',
     'runtime-close',
     'workspace-manifest',
@@ -33,7 +33,7 @@ function runEvidence(trace, checks) {
             retries += 1;
     }
     return {
-        safety_passed: SAFETY_CHECKS.every((name) => checks.find((check) => check.name === name)?.passed === true),
+        safety_passed: PROJECT_EVAL_SAFETY_CHECKS.every((name) => checks.find((check) => check.name === name)?.passed === true),
         permission_decisions,
         tool_errors,
         retries,
@@ -144,6 +144,7 @@ export async function runProjectEvalCase(options) {
             effectiveTools: allowedTools,
             runVerification: options.runVerification ?? false,
             praxisVersion: options.version,
+            buildIdentity: options.buildIdentity,
         });
     }
     catch (error) {
@@ -224,67 +225,114 @@ export async function runProjectEvalCase(options) {
             cleanupErrors.push(`Runtime cleanup failed: ${runtimeCloseError}`);
         }
     }
-    if (runtimeCompleted && !runtimeCloseError && !termination) {
-        for (const verifier of options.case.verification) {
-            const verifierStarted = Date.now();
-            try {
-                const result = await new BoundedProcessRunner({
-                    cwd: workspace.cwd,
-                    maxOutputBytes: 64 * 1024,
-                }).run({
-                    command: verifier.command,
-                    args: [...verifier.args],
-                    cwd: workspace.cwd,
-                    timeoutMs: verifier.timeoutSeconds * 1000,
-                    ...(options.signal === undefined ? {} : { signal: options.signal }),
-                    env: verifierEnvironment(options.case.execution.env, workspace.home, workspace.root),
-                    inheritEnvironment: false,
-                    redactExplicitEnvironment: true,
-                });
-                const passed = result.code === 0 && !result.timedOut;
-                const verifierError = result.timedOut
+    const addNotRunVerifier = (verifier, reason) => {
+        verifications.push({
+            schema_version: '1.1',
+            name: verifier.name,
+            required: true,
+            expect: 'pass',
+            status: 'not_run',
+            command: verifier.command,
+            argv: [...verifier.args],
+            exit_code: null,
+            timed_out: false,
+            stdout: '',
+            stderr: '',
+            truncated: false,
+            duration_ms: 0,
+            passed: false,
+            error: reason,
+        });
+        verifierChecks.push(check(`verifier:${verifier.name}`, false, reason));
+    };
+    let verifierInterrupted = false;
+    for (const verifier of options.case.verification) {
+        if (verifierInterrupted ||
+            !runtimeCompleted ||
+            runtimeCloseError ||
+            termination) {
+            addNotRunVerifier(verifier, verifierInterrupted
+                ? 'Verifier not run: previous verifier interrupted'
+                : runtimeCloseError
+                    ? 'Verifier not run: runtime cleanup failed'
+                    : termination
+                        ? `Verifier not run: runtime ${termination}`
+                        : 'Verifier not run: runtime did not complete');
+            continue;
+        }
+        const verifierStarted = Date.now();
+        try {
+            const result = await new BoundedProcessRunner({
+                cwd: workspace.cwd,
+                maxOutputBytes: 64 * 1024,
+            }).run({
+                command: verifier.command,
+                args: [...verifier.args],
+                cwd: workspace.cwd,
+                timeoutMs: verifier.timeoutSeconds * 1000,
+                ...(options.signal === undefined ? {} : { signal: options.signal }),
+                env: verifierEnvironment(options.case.execution.env, workspace.home, workspace.root),
+                inheritEnvironment: false,
+                redactExplicitEnvironment: true,
+            });
+            const interrupted = options.signal?.aborted === true;
+            if (interrupted) {
+                termination = 'interrupted';
+                verifierInterrupted = true;
+            }
+            const passed = result.code === 0 && !result.timedOut && !interrupted;
+            const verifierError = interrupted
+                ? 'Verifier interrupted'
+                : result.timedOut
                     ? 'Verifier timed out'
                     : result.code === 0
                         ? null
                         : `Verifier exited with code ${result.code}`;
-                verifications.push({
-                    schema_version: '1.0',
-                    name: verifier.name,
-                    command: verifier.command,
-                    argv: [...verifier.args],
-                    exit_code: result.code,
-                    timed_out: result.timedOut,
-                    stdout: result.stdout,
-                    stderr: result.stderr,
-                    truncated: result.truncated,
-                    duration_ms: Date.now() - verifierStarted,
-                    passed,
-                    error: verifierError,
-                });
-                verifierChecks.push(check(`verifier:${verifier.name}`, passed, passed ? 'Verifier passed' : (verifierError ?? 'Verifier failed')));
-            }
-            catch (error) {
-                const message = errorText(error);
-                if (options.signal?.aborted)
-                    termination = 'interrupted';
-                verifications.push({
-                    schema_version: '1.0',
-                    name: verifier.name,
-                    command: verifier.command,
-                    argv: [...verifier.args],
-                    exit_code: null,
-                    timed_out: false,
-                    stdout: '',
-                    stderr: '',
-                    truncated: false,
-                    duration_ms: Date.now() - verifierStarted,
-                    passed: false,
-                    error: message,
-                });
-                verifierChecks.push(check(`verifier:${verifier.name}`, false, message));
-                if (termination === 'interrupted')
-                    break;
+            const status = passed ? 'passed' : 'failed';
+            verifications.push({
+                schema_version: '1.1',
+                name: verifier.name,
+                required: true,
+                expect: 'pass',
+                status,
+                command: verifier.command,
+                argv: [...verifier.args],
+                exit_code: result.code,
+                timed_out: result.timedOut,
+                stdout: result.stdout,
+                stderr: result.stderr,
+                truncated: result.truncated,
+                duration_ms: Date.now() - verifierStarted,
+                passed,
+                error: verifierError,
+            });
+            verifierChecks.push(check(`verifier:${verifier.name}`, passed, passed ? 'Verifier passed' : (verifierError ?? 'Verifier failed')));
+        }
+        catch (error) {
+            const message = errorText(error);
+            const interrupted = options.signal?.aborted === true;
+            if (interrupted) {
+                termination = 'interrupted';
+                verifierInterrupted = true;
             }
+            verifications.push({
+                schema_version: '1.1',
+                name: verifier.name,
+                required: true,
+                expect: 'pass',
+                status: 'failed',
+                command: verifier.command,
+                argv: [...verifier.args],
+                exit_code: null,
+                timed_out: false,
+                stdout: '',
+                stderr: '',
+                truncated: false,
+                duration_ms: Date.now() - verifierStarted,
+                passed: false,
+                error: interrupted ? 'Verifier interrupted' : message,
+            });
+            verifierChecks.push(check(`verifier:${verifier.name}`, false, interrupted ? 'Verifier interrupted' : message));
         }
     }
     const verifiersPassed = verifierChecks.every((item) => item.passed);
@@ -374,6 +422,15 @@ export async function runProjectEvalCase(options) {
             : `Temporary workspace cleanup failed: ${tempCleanupError}`));
     const passed = checks.every((item) => item.passed);
     const evidence = runEvidence(trace, checks);
+    const verification = {
+        outcomes: verifications.map(({ name, status }) => ({
+            name,
+            required: true,
+            expect: 'pass',
+            status,
+        })),
+        satisfied: verifications.every((verification) => verification.status === 'passed'),
+    };
     const primaryError = runtimeError ??
         graderError ??
         verifications.find((verification) => verification.error)?.error ??
@@ -381,8 +438,9 @@ export async function runProjectEvalCase(options) {
         checks.find((item) => !item.passed)?.explanation ??
         null;
     const result = {
-        schema_version: '1.1',
+        schema_version: '1.2',
         case: options.case.name,
+        risk: options.case.risk,
         run: options.run,
         version: options.version,
         model: identity.model_id,
@@ -398,6 +456,7 @@ export async function runProjectEvalCase(options) {
         ...evidence,
         graders,
         checks,
+        verification,
         artifacts: {
             trace: 'trace.jsonl',
             workspace_diff: 'workspace-diff.json',

package/dist/evals/project-eval-schema.d.ts

@@ -1,8 +1,12 @@
 import type { EvalDeterministicGrader } from './eval-contract.js';
-export declare const PROJECT_EVAL_SCHEMA_VERSION = "1.0";
+export declare const PROJECT_EVAL_SCHEMA_VERSION = "1.1";
+export declare const PROJECT_EVAL_RISKS: readonly ["low", "medium", "high", "release"];
+export type ProjectEvalRisk = (typeof PROJECT_EVAL_RISKS)[number];
+export declare const PROJECT_EVAL_MAX_ITEMS = 256;
 export interface ProjectEvalCase {
-    schemaVersion: '1.0';
+    schemaVersion: '1.1';
     name: string;
+    risk: ProjectEvalRisk;
     dir: string;
     fixture: string;
     tags: readonly string[];
@@ -21,6 +25,8 @@ export interface ProjectEvalCase {
         command: string;
         args: readonly string[];
         timeoutSeconds: number;
+        required: true;
+        expect: 'pass';
     }[];
     graders: readonly EvalDeterministicGrader[];
     expect: {

package/dist/evals/project-eval-schema.js

@@ -2,9 +2,10 @@ import { lstat, readFile, readdir, realpath } from 'node:fs/promises';
 import { isAbsolute, join, resolve, sep } from 'node:path';
 import { parse as parseYaml } from 'yaml';
 import { Minimatch, minimatch } from 'minimatch';
-export const PROJECT_EVAL_SCHEMA_VERSION = '1.0';
+export const PROJECT_EVAL_SCHEMA_VERSION = '1.1';
+export const PROJECT_EVAL_RISKS = ['low', 'medium', 'high', 'release'];
 const NAME = /^[A-Za-z0-9][A-Za-z0-9._-]{0,127}$/u;
-const MAX_ITEMS = 256;
+export const PROJECT_EVAL_MAX_ITEMS = 256;
 const MAX_STRING = 16 * 1024;
 const MAX_BYTES = 1024 * 1024;
 const DEFAULT_PROJECT_EVAL_ALLOWED_TOOLS = ['Read', 'Glob', 'Grep'];
@@ -17,14 +18,14 @@ function validateBounds(value, label = 'case', depth = 0, state = { nodes: 0 })
     if (typeof value === 'string' && value.length > MAX_STRING)
         throw new Error(`${label} contains oversized string`);
     if (Array.isArray(value)) {
-        if (value.length > MAX_ITEMS)
+        if (value.length > PROJECT_EVAL_MAX_ITEMS)
             throw new Error(`${label} contains oversized array`);
         for (const item of value)
             validateBounds(item, label, depth + 1, state);
     }
     else if (value && typeof value === 'object') {
         const entries = Object.entries(value);
-        if (entries.length > MAX_ITEMS)
+        if (entries.length > PROJECT_EVAL_MAX_ITEMS)
             throw new Error(`${label} contains oversized object`);
         for (const [key, item] of entries) {
             if (key.length > 256)
@@ -64,7 +65,7 @@ function list(v, label) {
     if (v === undefined)
         return [];
     if (!Array.isArray(v) ||
-        v.length > MAX_ITEMS ||
+        v.length > PROJECT_EVAL_MAX_ITEMS ||
         v.some((x) => typeof x !== 'string' || !x || x.length > MAX_STRING))
         throw new Error(`${label} must be a bounded string array`);
     return v.map((x) => String(x));
@@ -216,6 +217,7 @@ export function parseProjectEvalCase(raw, dir) {
     strict(r, [
         'schema_version',
         'name',
+        'risk',
         'tags',
         'runs',
         'fixture',
@@ -224,11 +226,13 @@ export function parseProjectEvalCase(raw, dir) {
         'graders',
         'expect',
     ], 'case');
-    if (r.schema_version !== '1.0')
+    if (r.schema_version !== '1.1')
         throw new Error(`Unsupported schema_version: ${String(r.schema_version)}`);
     const name = str(r.name, 'name');
     if (!NAME.test(name))
         throw new Error('name must be a safe eval identifier');
+    if (!PROJECT_EVAL_RISKS.includes(r.risk))
+        throw new Error('risk must be one of: low, medium, high, release');
     const ex = obj(r.execution, 'execution');
     strict(ex, [
         'prompt',
@@ -252,27 +256,39 @@ export function parseProjectEvalCase(raw, dir) {
         if (!/^EVAL_[A-Z0-9_]+$/u.test(k) || typeof v !== 'string')
             throw new Error(`Invalid eval environment variable: ${k}`);
     const vr = r.verification === undefined ? [] : r.verification;
-    if (!Array.isArray(vr) || vr.length > MAX_ITEMS)
+    if (!Array.isArray(vr) || vr.length > PROJECT_EVAL_MAX_ITEMS)
         throw new Error('verification must be a bounded array');
     const verification = vr.map((x, i) => {
         const q = obj(x, `verification[${i}]`);
-        strict(q, ['name', 'command', 'args', 'timeout_seconds'], `verification[${i}]`);
+        strict(q, ['name', 'command', 'args', 'timeout_seconds', 'required', 'expect'], `verification[${i}]`);
+        const verifierName = str(q.name, `verification[${i}].name`);
+        if (!NAME.test(verifierName))
+            throw new Error(`verification[${i}].name must be a safe eval identifier`);
+        if (q.required !== true)
+            throw new Error(`verification[${i}].required must be true`);
+        if (q.expect !== 'pass')
+            throw new Error(`verification[${i}].expect must be pass`);
         return {
-            name: str(q.name, `verification[${i}].name`),
+            name: verifierName,
             command: str(q.command, `verification[${i}].command`),
             args: list(q.args, `verification[${i}].args`),
             timeoutSeconds: integer(q.timeout_seconds, `verification[${i}].timeout_seconds`, 120, 1, 3600),
+            required: true,
+            expect: 'pass',
         };
     });
+    if (new Set(verification.map((item) => item.name)).size !== verification.length)
+        throw new Error('Project eval verifier names must be unique');
     const gradersRaw = r.graders === undefined ? [] : r.graders;
-    if (!Array.isArray(gradersRaw) || gradersRaw.length > MAX_ITEMS)
+    if (!Array.isArray(gradersRaw) || gradersRaw.length > PROJECT_EVAL_MAX_ITEMS)
         throw new Error('graders must be a bounded array');
     const graders = gradersRaw.map(projectGrader);
     if (new Set(graders.map((item) => item.name)).size !== graders.length)
         throw new Error('Project grader names must be unique');
     return {
-        schemaVersion: '1.0',
+        schemaVersion: '1.1',
         name,
+        risk: r.risk,
         dir,
         fixture,
         tags: list(r.tags, 'tags'),

package/dist/evals/project-eval.d.ts

@@ -1,10 +1,13 @@
 import type { ModelUsage } from '../core/runtime.js';
 import type { IdentifiedEvalRuntimeFactory } from './eval-contract.js';
-import { type ProjectEvalRunResult } from './project-eval-runner.js';
+import { type ProjectEvalCheckSummary, type ProjectEvalVerificationEvidence, type ProjectEvalRunResult } from './project-eval-runner.js';
+export type { ProjectEvalCheckSummary, ProjectEvalVerifierOutcome, ProjectEvalVerificationEvidence, } from './project-eval-runner.js';
 import { type ProjectEvalIdentity } from './project-eval-identity.js';
+import type { PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
 export declare const PROJECT_EVAL_HELP = "Usage: praxis eval [options] <target>\n\nRun deterministic project outcome evaluations in isolated workspaces.\n\nOptions:\n  --case <glob>          Filter case names\n  --tag <tag[,tag]>      Filter tags; repeatable\n  --runs <1..50>         Override run count\n  --model <model>        Override model\n  --allow-tools <rules>  Grant gated tools; comma-separated and repeatable\n  --run-verification     Enable verifier subprocesses\n  --output-dir <dir>     Write artifacts to this directory\n  --keep-temp            Preserve temporary workspaces\n  --json                 Print exactly one aggregate JSON value\n  --verbose              Print run progress to stderr\n  -h, --help             Display help\n\nUse praxis eval compare --help to compare two completed aggregate artifacts.";
 export interface ProjectEvalDependencies {
     runtimeFactory: IdentifiedEvalRuntimeFactory;
+    loadBuildIdentity: () => Promise<PraxisBuildIdentity>;
     version?: string;
     configRoot: string;
 }
@@ -30,8 +33,10 @@ export interface ProjectEvalUsageTotals {
     web_search_requests: number;
 }
 export interface ProjectEvalRunSummary {
+    schema_version: '1.2';
     case: string;
     run: number;
+    risk: ProjectEvalRunResult['risk'];
     model: string;
     passed: boolean;
     score: 0 | 1;
@@ -52,9 +57,11 @@ export interface ProjectEvalRunSummary {
     error: string | null;
     artifact_dir: string;
     identity: ProjectEvalIdentity;
+    verification: ProjectEvalVerificationEvidence;
+    checks: readonly ProjectEvalCheckSummary[];
 }
 export interface ProjectEvalAggregate {
-    schema_version: '1.1';
+    schema_version: '1.2';
     version: string;
     start: string;
     duration_ms: number;
@@ -84,6 +91,19 @@ export interface ProjectEvalAggregate {
     };
     tool_errors: number;
     retries: number;
+    verification_totals: {
+        declared: number;
+        passed: number;
+        failed: number;
+        not_run: number;
+        satisfied_runs: number;
+        unsatisfied_runs: number;
+    };
+    risk_tiers: Record<ProjectEvalRunResult['risk'], {
+        runs: number;
+        passed: number;
+        failed: number;
+    }>;
     terminations: {
         completed: number;
         timeout: number;

package/dist/evals/project-eval.js

@@ -118,8 +118,10 @@ function aggregateUsage(results) {
 }
 function runSummary(result, outputDirectory) {
     return {
+        schema_version: '1.2',
         case: result.case,
         run: result.run,
+        risk: result.risk,
         model: result.model,
         passed: result.passed,
         score: result.score,
@@ -136,6 +138,8 @@ function runSummary(result, outputDirectory) {
         error: result.error,
         artifact_dir: relative(outputDirectory, join(outputDirectory, result.case, `run-${result.run}`)).replaceAll('\\', '/'),
         identity: result.identity,
+        verification: result.verification,
+        checks: result.checks.map(({ name, passed }) => ({ name, passed })),
     };
 }
 export async function executeProjectEvalCommand(argv, io, dependencies, callerCwd = process.cwd(), signal) {
@@ -150,6 +154,7 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
         throw new Error('eval requires one target');
     const target = resolve(callerCwd, options.target);
     const cases = await discoverProjectEvalCases(target, options.caseGlob, options.tags);
+    const buildIdentity = await dependencies.loadBuildIdentity();
     const outputDirectory = options.outputDir
         ? resolve(callerCwd, options.outputDir)
         : join(dependencies.configRoot, 'evals', 'results', new Date().toISOString().replaceAll(':', '-'));
@@ -173,6 +178,7 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
                 runVerification: options.runVerification,
                 outputDir: outputDirectory,
                 version: dependencies.version ?? 'unknown',
+                buildIdentity,
                 ...(signal === undefined ? {} : { signal }),
             });
             results.push(result);
@@ -187,8 +193,33 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
     const passed = results.filter((result) => result.passed).length;
     const knownCostResults = results.filter((result) => result.cost_known);
     const summaries = results.map((result) => runSummary(result, outputDirectory));
+    const verificationTotals = {
+        declared: summaries.reduce((total, summary) => total + summary.verification.outcomes.length, 0),
+        passed: summaries.reduce((total, summary) => total +
+            summary.verification.outcomes.filter(({ status }) => status === 'passed').length, 0),
+        failed: summaries.reduce((total, summary) => total +
+            summary.verification.outcomes.filter(({ status }) => status === 'failed').length, 0),
+        not_run: summaries.reduce((total, summary) => total +
+            summary.verification.outcomes.filter(({ status }) => status === 'not_run').length, 0),
+        satisfied_runs: summaries.filter((summary) => summary.verification.satisfied).length,
+        unsatisfied_runs: summaries.filter((summary) => !summary.verification.satisfied).length,
+    };
+    const riskTiers = {
+        low: { runs: 0, passed: 0, failed: 0 },
+        medium: { runs: 0, passed: 0, failed: 0 },
+        high: { runs: 0, passed: 0, failed: 0 },
+        release: { runs: 0, passed: 0, failed: 0 },
+    };
+    for (const summary of summaries) {
+        const tier = riskTiers[summary.risk];
+        tier.runs += 1;
+        if (summary.passed)
+            tier.passed += 1;
+        else
+            tier.failed += 1;
+    }
     const aggregate = {
-        schema_version: '1.1',
+        schema_version: '1.2',
         version: dependencies.version ?? 'unknown',
         start: new Date(started).toISOString(),
         duration_ms: Date.now() - started,
@@ -224,6 +255,8 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
         },
         tool_errors: results.reduce((total, result) => total + result.tool_errors, 0),
         retries: results.reduce((total, result) => total + result.retries, 0),
+        verification_totals: verificationTotals,
+        risk_tiers: riskTiers,
         terminations: {
             completed: results.filter((result) => result.termination === null).length,
             timeout: results.filter((result) => result.termination === 'timeout')

package/dist/platform/praxis-build-identity.d.ts

@@ -0,0 +1,14 @@
+export declare const PRAXIS_BUILD_IDENTITY_SCHEMA_VERSION: "1.0";
+export interface PraxisBuildIdentity {
+    schema_version: typeof PRAXIS_BUILD_IDENTITY_SCHEMA_VERSION;
+    source_revision: `git:${string}` | 'unavailable';
+    source_dirty: boolean | null;
+    artifact_sha256: `sha256:${string}`;
+}
+export declare function writePraxisBuildIdentity({ sourceRoot, outputRoot, }: {
+    sourceRoot: string;
+    outputRoot: string;
+}): Promise<PraxisBuildIdentity>;
+export declare function validatePraxisBuildIdentity(value: unknown): PraxisBuildIdentity;
+export declare function loadPraxisBuildIdentity(outputRoot?: string): Promise<PraxisBuildIdentity>;
+//# sourceMappingURL=praxis-build-identity.d.ts.map