praxis-agent 0.67.2 → 0.69.0

package/dist/evals/project-eval-runner.d.ts

@@ -79,6 +79,8 @@ interface ProjectEvalRunOptions {
     factory: IdentifiedEvalRuntimeFactory;
     run: number;
     allowTools?: readonly string[];
+    provider?: string;
+    profile?: string;
     model?: string;
     keepTemp?: boolean;
     runVerification?: boolean;

package/dist/evals/project-eval-runner.js

@@ -124,6 +124,10 @@ export async function runProjectEvalCase(options) {
         cwd: workspace.cwd,
         configRoot: workspace.config,
         home: workspace.home,
+        ...(options.provider === undefined ? {} : { provider: options.provider }),
+        ...(options.profile === undefined
+            ? {}
+            : { providerProfile: options.profile }),
         maxTurns: options.case.execution.maxTurns,
         pluginDirectories: [],
         allowedTools,

package/dist/evals/project-eval.d.ts

@@ -4,7 +4,7 @@ import { type ProjectEvalCheckSummary, type ProjectEvalVerificationEvidence, typ
 export type { ProjectEvalCheckSummary, ProjectEvalVerifierOutcome, ProjectEvalVerificationEvidence, } from './project-eval-runner.js';
 import { type ProjectEvalIdentity } from './project-eval-identity.js';
 import type { PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
-export declare const PROJECT_EVAL_HELP = "Usage: praxis eval [options] <target>\n\nRun deterministic project outcome evaluations in isolated workspaces.\n\nOptions:\n  --case <glob>          Filter case names\n  --tag <tag[,tag]>      Filter tags; repeatable\n  --runs <1..50>         Override run count\n  --model <model>        Override model\n  --allow-tools <rules>  Grant gated tools; comma-separated and repeatable\n  --run-verification     Enable verifier subprocesses\n  --output-dir <dir>     Write artifacts to this directory\n  --keep-temp            Preserve temporary workspaces\n  --json                 Print exactly one aggregate JSON value\n  --verbose              Print run progress to stderr\n  -h, --help             Display help\n\nUse praxis eval compare --help to compare two completed aggregate artifacts.";
+export declare const PROJECT_EVAL_HELP = "Usage: praxis eval [options] <target>\n\nRun deterministic project outcome evaluations in isolated workspaces.\n\nOptions:\n  --case <glob>          Filter case names\n  --tag <tag[,tag]>      Filter tags; repeatable\n  --runs <1..50>         Override run count\n  --model <model>        Override model\n  --provider <id>        Override provider\n  --profile <id>         Override provider profile\n  --allow-tools <rules>  Grant gated tools; comma-separated and repeatable\n  --run-verification     Enable verifier subprocesses\n  --output-dir <dir>     Write artifacts to this directory\n  --keep-temp            Preserve temporary workspaces\n  --json                 Print exactly one aggregate JSON value\n  --verbose              Print run progress to stderr\n  -h, --help             Display help\n\nUse praxis eval compare --help to compare two completed aggregate artifacts.";
 export interface ProjectEvalDependencies {
     runtimeFactory: IdentifiedEvalRuntimeFactory;
     loadBuildIdentity: () => Promise<PraxisBuildIdentity>;
@@ -17,6 +17,8 @@ export interface ProjectEvalOptions {
     tags: string[];
     runs?: number;
     model?: string;
+    provider?: string;
+    profile?: string;
     allowTools: string[];
     runVerification: boolean;
     outputDir?: string;

package/dist/evals/project-eval.js

@@ -13,6 +13,8 @@ Options:
   --tag <tag[,tag]>      Filter tags; repeatable
   --runs <1..50>         Override run count
   --model <model>        Override model
+  --provider <id>        Override provider
+  --profile <id>         Override provider profile
   --allow-tools <rules>  Grant gated tools; comma-separated and repeatable
   --run-verification     Enable verifier subprocesses
   --output-dir <dir>     Write artifacts to this directory
@@ -60,6 +62,8 @@ export function parseProjectEvalOptions(argv) {
             options.verbose = true;
         else if (value === '--case' ||
             value === '--model' ||
+            value === '--provider' ||
+            value === '--profile' ||
             value === '--output-dir' ||
             value === '--runs' ||
             value === '--tag' ||
@@ -70,6 +74,10 @@ export function parseProjectEvalOptions(argv) {
                 options.caseGlob = selected;
             else if (value === '--model')
                 options.model = selected;
+            else if (value === '--provider')
+                options.provider = selected;
+            else if (value === '--profile')
+                options.profile = selected;
             else if (value === '--output-dir')
                 options.outputDir = selected;
             else if (value === '--runs') {
@@ -143,6 +151,10 @@ function runSummary(result, outputDirectory) {
     };
 }
 export async function executeProjectEvalCommand(argv, io, dependencies, callerCwd = process.cwd(), signal) {
+    if (argv[0] === 'qualify') {
+        const { executeHeldOutQualificationCommand } = await import('./held-out-qualification.js');
+        return executeHeldOutQualificationCommand(argv.slice(1), io, dependencies, callerCwd, signal);
+    }
     if (argv[0] === 'compare')
         return executeProjectEvalCompareCommand(argv.slice(1), io, callerCwd, signal);
     const options = parseProjectEvalOptions(argv);
@@ -174,6 +186,10 @@ export async function executeProjectEvalCommand(argv, io, dependencies, callerCw
                 run: runIndex,
                 allowTools: options.allowTools,
                 ...(options.model === undefined ? {} : { model: options.model }),
+                ...(options.provider === undefined
+                    ? {}
+                    : { provider: options.provider }),
+                ...(options.profile === undefined ? {} : { profile: options.profile }),
                 keepTemp: options.keepTemp,
                 runVerification: options.runVerification,
                 outputDir: outputDirectory,

package/package.json

@@ -1,6 +1,6 @@
 {
   "name": "praxis-agent",
-  "version": "0.67.2",
+  "version": "0.69.0",
   "description": "Local-first, single-user general agent for the command line.",
   "license": "MIT",
   "author": "wuqisen",
@@ -62,7 +62,9 @@
     "verify:ci-coverage": "node scripts/verify-ci-coverage.mjs",
     "verify:fixture-contracts": "node scripts/verify-fixture-contracts.mjs",
     "test:fixtures": "node scripts/run-fixture-contracts.mjs",
-    "test:eval:baseline": "vitest run src/evals/coding-baseline.test.ts src/evals/project-eval-comparison.test.ts src/evals/apply-patch-admission.test.ts src/evals/lsp-diagnostics-admission.test.ts src/evals/glob-ripgrep-admission.test.ts"
+    "test:eval:baseline": "vitest run src/evals/coding-baseline.test.ts src/evals/project-eval-comparison.test.ts src/evals/apply-patch-admission.test.ts src/evals/lsp-diagnostics-admission.test.ts src/evals/glob-ripgrep-admission.test.ts",
+    "test:eval:held-out-contract": "vitest run src/evals/held-out-corpus.test.ts",
+    "test:eval:held-out-qualification": "vitest run src/evals/held-out-qualification.test.ts"
   },
   "engines": {
     "node": ">=24"