praxis-agent 0.46.4 → 0.47.0

package/README.md

@@ -100,6 +100,10 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
 
 ## What Praxis provides
 
+- **Outcome-driven evaluation** — `praxis eval <target>` runs contained cases
+  in isolated workspaces, requires explicit verifier authorization, and writes
+  versioned artifacts locally; usage and cost remain explicitly available or
+  unknown.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,

package/dist/cli-runtime.d.ts

@@ -22,6 +22,7 @@ import { launchTmuxWorktree } from './platform/tmux-worktree.js';
 import { type CliControls, type CliRuntimeInfo, type CliElicitationRequest, type CliElicitationResult } from './cli/protocol.js';
 import { executeProviderAuthCommand } from './cli/provider-auth-command.js';
 import { type PluginEvalDependencies } from './plugins/claude-plugin-eval.js';
+import { type ProjectEvalDependencies } from './evals/project-eval.js';
 import { type SelfUpdateResult } from './maintenance/self-update.js';
 export { parseContextEnvironment, parseProviderEnvironment };
 export interface CliIO {
@@ -153,6 +154,7 @@ export interface CliDependencies extends InteractiveServiceFactory {
         statePath?: string;
     }): Promise<ModelProvider>;
     pluginEval?: PluginEvalDependencies;
+    projectEval?: ProjectEvalDependencies;
     runInteractive?(options: {
         agent?: string;
         controls?: CliControls;

package/dist/cli-runtime.js

@@ -70,6 +70,7 @@ import { executeProviderAuthCommand } from './cli/provider-auth-command.js';
 import { describeClaudePlugin, initClaudePlugin, installClaudePlugin, loadClaudePlugins, readPluginRegistry, setClaudePluginEnabled, uninstallClaudePlugin, updateClaudePlugin, validateClaudePlugin, } from './plugins/claude-plugin-runtime.js';
 import { addClaudeMarketplace, disableAllNativePlugins, installClaudeMarketplacePlugin, listClaudeMarketplaceAvailablePlugins, listNativePluginRecords, readClaudeKnownMarketplaces, removeClaudeMarketplace, setNativePluginEnabled, saveClaudePluginConfig, uninstallNativePlugin, updateClaudeMarketplace, updateNativePlugin, validateClaudeMarketplace, } from './plugins/claude-plugin-marketplace.js';
 import { executeClaudePluginEvalCommand, PLUGIN_EVAL_HELP, } from './plugins/claude-plugin-eval.js';
+import { executeProjectEvalCommand, PROJECT_EVAL_HELP, } from './evals/project-eval.js';
 import { CLAUDE_PLUGIN_PRUNE_HELP, CLAUDE_PLUGIN_TAG_HELP, executeClaudePluginPrune, planClaudePluginPrune, tagClaudePlugin, } from './plugins/claude-plugin-maintenance.js';
 import { formatDoctorReport, runDoctor } from './maintenance/doctor.js';
 import { runSelfUpdate, } from './maintenance/self-update.js';
@@ -123,6 +124,7 @@ Usage:
   praxis mcp <list|get|add|add-json|remove|reset-project-choices|login|logout|serve> ...
   praxis auto-mode <config|defaults|critique>
   praxis plugin|plugins <details|list|install|uninstall|enable|disable|update|init|prune|tag|validate|marketplace> ...
+  praxis eval [options] <target>
   praxis doctor [--json]
   praxis auth <status|set-key|login|logout> ...
   praxis import [options] [source]
@@ -2282,7 +2284,7 @@ const defaultPluginEvalRuntimeFactory = {
                 dataPlane: options.dataPlane,
                 sessionPersistence: false,
                 maxTurns: options.maxTurns,
-                pluginDirectories: [...options.pluginDirectories],
+                pluginDirectories: [...(options.pluginDirectories ?? [])],
                 addDirectories: [...options.addDirs],
                 allowedTools: [...options.allowedTools],
                 disallowedTools: [],
@@ -2305,6 +2307,7 @@ const defaultPluginEvalRuntimeFactory = {
                     ...(result.costUsd === undefined
                         ? {}
                         : { costUsd: result.costUsd }),
+                    usage: result.usage,
                 };
             },
             close: () => service.close?.() ?? Promise.resolve(),
@@ -2523,6 +2526,11 @@ export function createDefaultDependencies(entrypoint = fileURLToPath(import.meta
             runtimeFactory: defaultPluginEvalRuntimeFactory,
             judge: defaultPluginEvalJudge,
         },
+        projectEval: {
+            runtimeFactory: defaultPluginEvalRuntimeFactory,
+            version: VERSION,
+            configRoot: resolveDataPlaneRoot(),
+        },
         cliPath: entrypoint,
         runInteractive: async ({ agent, controls, initialPrompt, resume, signal, }) => {
             const { runInteractive } = await import('./cli/interactive.js');
@@ -4899,6 +4907,7 @@ async function executeTeamCommand(args, io, dependencies, signal) {
 }
 function specialCommandIndex(argv) {
     const commands = new Set([
+        'eval',
         'team',
         'plugin',
         'plugins',
@@ -4963,7 +4972,9 @@ async function execute(argv, io, dependencies, signal) {
         return executeTeamCommand(argv.slice(commandIndex), io, dependencies, signal);
     }
     const specialPrefix = commandIndex > 0 &&
-        (argv[commandIndex] === 'plugin' || argv[commandIndex] === 'plugins')
+        (argv[commandIndex] === 'plugin' ||
+            argv[commandIndex] === 'plugins' ||
+            argv[commandIndex] === 'eval')
         ? parseCliInvocation([...argv.slice(0, commandIndex), '__probe__'])
         : undefined;
     const special = commandIndex >= 0
@@ -4971,6 +4982,22 @@ async function execute(argv, io, dependencies, signal) {
             args: argv.slice(commandIndex),
         }
         : { args: [...argv] };
+    if (special.args[0] === 'eval') {
+        if (special.args
+            .slice(1)
+            .some((value) => value === '-h' || value === '--help')) {
+            io.stdout(PROJECT_EVAL_HELP);
+            return 0;
+        }
+        if (!dependencies.projectEval)
+            throw new Error('Project eval unavailable');
+        return executeProjectEvalCommand([
+            ...(specialPrefix?.model === undefined
+                ? []
+                : ['--model', specialPrefix.model]),
+            ...special.args.slice(1),
+        ], io, dependencies.projectEval, process.cwd(), signal);
+    }
     if ((special.args[0] === 'plugin' || special.args[0] === 'plugins') &&
         special.args[1] === 'eval') {
         if (!dependencies.pluginEval)

package/dist/evals/eval-contract.d.ts

@@ -0,0 +1,102 @@
+import type { ModelUsage, RuntimeEvent } from '../core/runtime.js';
+import type { DataPlane } from '../persistence/data-plane.js';
+export interface EvalRuntime {
+    run(prompt: string, signal: AbortSignal): Promise<{
+        text: string;
+        turns: number;
+        costUsd?: number;
+        usage?: ModelUsage;
+    }>;
+    close?(): Promise<void>;
+}
+export interface EvalRuntimeFactoryOptions {
+    dataPlane: DataPlane;
+    cwd: string;
+    configRoot: string;
+    home: string;
+    model?: string;
+    maxTurns: number;
+    pluginDirectories?: readonly string[];
+    allowedTools: readonly string[];
+    appendSystemPrompt?: string;
+    historyFile?: string;
+    addDirs: readonly string[];
+    env: Readonly<Record<string, string>>;
+    eventSink(event: RuntimeEvent): void;
+}
+export interface EvalRuntimeFactory {
+    create(options: EvalRuntimeFactoryOptions): Promise<EvalRuntime>;
+}
+export interface EvalTraceEvent {
+    type: string;
+    tool?: string;
+    input?: Record<string, unknown>;
+    [key: string]: unknown;
+}
+export type EvalFocus = 'last_message' | 'trace' | 'files' | {
+    source: 'file';
+    path: string;
+};
+export type EvalArm = 'with-only' | 'both';
+export type EvalToolMatch = string | {
+    tool: string;
+    input_match?: Record<string, unknown>;
+};
+export interface EvalGrader {
+    name: string;
+    weight: number;
+    arm?: EvalArm;
+}
+export type EvalGraderBase = EvalGrader;
+export type EvalDeterministicGrader = (EvalGrader & {
+    type: 'regex';
+    target: EvalFocus;
+    pattern: string;
+    flags: string;
+    match: 'contains' | 'not_contains' | `count:${number}`;
+}) | (EvalGrader & {
+    type: 'tool_order';
+    before: EvalToolMatch;
+    after: EvalToolMatch;
+}) | (EvalGrader & {
+    type: 'tool_used';
+    tool: string;
+    input_match?: Record<string, unknown>;
+    min: number;
+    max: number;
+}) | (EvalGrader & {
+    type: 'file_exists';
+    path: string;
+    exists: boolean;
+});
+export interface EvalRunArtifacts {
+    lastMessage: string;
+    trace: readonly EvalTraceEvent[];
+    cwd: string;
+}
+export interface EvalGraderResult {
+    name: string;
+    passed: boolean;
+    weight: number;
+    explanation: string;
+    judge_votes?: readonly boolean[];
+    evidence?: string;
+    with_only?: boolean;
+}
+export interface EvalJudge {
+    vote(request: {
+        criteria: string;
+        focus: string;
+        baseline?: string;
+        model: string;
+        signal?: AbortSignal;
+    }): Promise<{
+        passed: boolean;
+        explanation?: string;
+        costUsd: number;
+    }>;
+}
+export declare const DEFAULT_EVAL_ALLOWED_TOOLS: readonly ["Read", "Glob", "Grep", "Skill"];
+export declare function resolveEvalAllowedTools(requested: readonly string[], operatorGrants: readonly string[]): string[];
+export declare function normalizeEvalTraceEvent(event: RuntimeEvent): EvalTraceEvent;
+//# sourceMappingURL=eval-contract.d.ts.map

package/dist/evals/eval-contract.js

@@ -0,0 +1,57 @@
+export const DEFAULT_EVAL_ALLOWED_TOOLS = [
+    'Read',
+    'Glob',
+    'Grep',
+    'Skill',
+];
+function toolName(rule) {
+    const i = rule.indexOf('(');
+    return i < 0 ? rule : rule.slice(0, i);
+}
+function gatedTool(rule) {
+    const name = toolName(rule);
+    return (['Bash', 'Write', 'Edit', 'NotebookEdit', 'WebFetch', 'WebSearch'].includes(name) || name.startsWith('mcp__'));
+}
+function grants(requested, allowed) {
+    const name = toolName(requested);
+    const wildcard = (pattern) => {
+        const parts = pattern.split('*');
+        if (parts.length === 1 || !requested.startsWith(parts[0] ?? ''))
+            return false;
+        let cursor = parts[0]?.length ?? 0;
+        for (const part of parts.slice(1, -1)) {
+            const index = requested.indexOf(part, cursor);
+            if (index < 0)
+                return false;
+            cursor = index + part.length;
+        }
+        const last = parts.at(-1) ?? '';
+        return last.length === 0 || requested.slice(cursor).endsWith(last);
+    };
+    return allowed.some((grant) => grant === requested || grant === name || wildcard(grant));
+}
+export function resolveEvalAllowedTools(requested, operatorGrants) {
+    const selected = requested.length ? requested : DEFAULT_EVAL_ALLOWED_TOOLS;
+    for (const rule of selected)
+        if (gatedTool(rule) && !grants(rule, operatorGrants))
+            throw new Error(`Eval case requests gated tool ${rule}; grant it with --allow-tools`);
+    return [...new Set(selected)];
+}
+export function normalizeEvalTraceEvent(event) {
+    if (event.type === 'tool-call')
+        return {
+            type: 'tool-call',
+            tool: event.call.name,
+            input: event.call.input,
+            id: event.call.id,
+        };
+    if (event.type === 'tool-result')
+        return {
+            type: 'tool-result',
+            callId: event.callId,
+            content: event.content,
+            isError: event.isError,
+        };
+    return event;
+}
+//# sourceMappingURL=eval-contract.js.map

package/dist/evals/eval-graders.d.ts

@@ -0,0 +1,6 @@
+import { type EvalDeterministicGrader, type EvalGraderResult, type EvalRunArtifacts } from './eval-contract.js';
+export declare function gradeDeterministicEvalRun(options: {
+    graders: readonly EvalDeterministicGrader[];
+    artifacts: EvalRunArtifacts;
+}): Promise<EvalGraderResult[]>;
+//# sourceMappingURL=eval-graders.d.ts.map

package/dist/evals/eval-graders.js

@@ -0,0 +1,94 @@
+import { glob, readFile, realpath } from 'node:fs/promises';
+import { resolve, sep } from 'node:path';
+import { minimatch } from 'minimatch';
+function subset(actual, expected) {
+    if (expected && typeof expected === 'object' && !Array.isArray(expected)) {
+        if (!actual || typeof actual !== 'object' || Array.isArray(actual))
+            return false;
+        return Object.entries(expected).every(([key, value]) => subset(actual[key], value));
+    }
+    return Object.is(actual, expected);
+}
+function matches(event, match) {
+    const selected = typeof match === 'string' ? { tool: match } : match;
+    return (event.type === 'tool-call' &&
+        event.tool === selected.tool &&
+        (!selected.input_match || subset(event.input, selected.input_match)));
+}
+async function containedPath(root, candidate) {
+    const rootPath = await realpath(resolve(root));
+    const path = await realpath(resolve(rootPath, candidate));
+    if (path !== rootPath && !path.startsWith(`${rootPath}${sep}`))
+        throw new Error(`Grader path escapes workspace: ${candidate}`);
+    return path;
+}
+async function focusText(focus, artifacts) {
+    if (focus === 'last_message')
+        return artifacts.lastMessage;
+    if (focus === 'trace')
+        return artifacts.trace.map((item) => JSON.stringify(item)).join('\n');
+    if (focus === 'files') {
+        const names = [];
+        for await (const path of glob('**/*', {
+            cwd: artifacts.cwd,
+            exclude: ['node_modules/**', '.git/**'],
+        }))
+            names.push(path);
+        return names.sort().join('\n');
+    }
+    return readFile(await containedPath(artifacts.cwd, focus.path), 'utf8');
+}
+export async function gradeDeterministicEvalRun(options) {
+    const results = [];
+    for (const grader of options.graders) {
+        let passed = false;
+        let evidence = '';
+        if (grader.type === 'regex') {
+            const target = await focusText(grader.target, options.artifacts);
+            const expression = new RegExp(grader.pattern, grader.flags.includes('g') ? grader.flags : `${grader.flags}g`);
+            const count = [...target.matchAll(expression)].length;
+            passed =
+                grader.match === 'contains'
+                    ? count > 0
+                    : grader.match === 'not_contains'
+                        ? count === 0
+                        : count === Number(grader.match.slice(6));
+            evidence = `matches=${count}`;
+        }
+        else if (grader.type === 'tool_used') {
+            const count = options.artifacts.trace.filter((event) => matches(event, {
+                tool: grader.tool,
+                ...(grader.input_match ? { input_match: grader.input_match } : {}),
+            })).length;
+            passed = count >= grader.min && count <= grader.max;
+            evidence = `uses=${count}`;
+        }
+        else if (grader.type === 'tool_order') {
+            const before = options.artifacts.trace.findIndex((event) => matches(event, grader.before));
+            const after = options.artifacts.trace.findIndex((event, index) => index > before && matches(event, grader.after));
+            passed = before >= 0 && after > before;
+            evidence = `before=${before},after=${after}`;
+        }
+        else {
+            let count = 0;
+            for await (const path of glob('**/*', { cwd: options.artifacts.cwd })) {
+                if (!minimatch(path, grader.path))
+                    continue;
+                await containedPath(options.artifacts.cwd, path);
+                count += 1;
+            }
+            passed = grader.exists ? count > 0 : count === 0;
+            evidence = `files=${count}`;
+        }
+        results.push({
+            name: grader.name,
+            passed,
+            weight: grader.weight,
+            explanation: passed ? 'passed' : 'failed',
+            evidence,
+            ...(grader.arm === 'with-only' ? { with_only: true } : {}),
+        });
+    }
+    return results;
+}
+//# sourceMappingURL=eval-graders.js.map

package/dist/evals/project-eval-runner.d.ts

@@ -0,0 +1,59 @@
+import type { ModelUsage } from '../core/runtime.js';
+import { type EvalGraderResult, type EvalRuntimeFactory } from './eval-contract.js';
+import type { ProjectEvalCase } from './project-eval-schema.js';
+export type ProjectEvalTermination = 'timeout' | 'interrupted' | null;
+export interface ProjectEvalVerificationResult {
+    schema_version: '1.0';
+    name: string;
+    command: string;
+    argv: readonly string[];
+    exit_code: number | null;
+    timed_out: boolean;
+    stdout: string;
+    stderr: string;
+    truncated: boolean;
+    duration_ms: number;
+    passed: boolean;
+    error: string | null;
+}
+export interface ProjectEvalRunResult {
+    schema_version: '1.0';
+    case: string;
+    run: number;
+    version: string;
+    model: string | null;
+    passed: boolean;
+    score: 0 | 1;
+    turns: number;
+    usage: ModelUsage | null;
+    cost_usd: number | null;
+    cost_known: boolean;
+    duration_ms: number;
+    termination: ProjectEvalTermination;
+    graders: readonly EvalGraderResult[];
+    checks: readonly EvalGraderResult[];
+    artifacts: {
+        trace: 'trace.jsonl';
+        workspace_diff: 'workspace-diff.json';
+        verification: 'verification.json';
+        result: 'result.json';
+    };
+    error: string | null;
+    cleanup_errors: readonly string[];
+    temp_root: string | null;
+}
+interface ProjectEvalRunOptions {
+    case: ProjectEvalCase;
+    factory: EvalRuntimeFactory;
+    run: number;
+    allowTools?: readonly string[];
+    model?: string;
+    keepTemp?: boolean;
+    runVerification?: boolean;
+    outputDir: string;
+    version: string;
+    signal?: AbortSignal;
+}
+export declare function runProjectEvalCase(options: ProjectEvalRunOptions): Promise<ProjectEvalRunResult>;
+export {};
+//# sourceMappingURL=project-eval-runner.d.ts.map