praxis-agent 0.68.0 → 0.69.0

package/README.md

@@ -144,6 +144,26 @@ troubleshooting. Run `praxis --help` for the authoritative command surface.
   safety-rate regression, every candidate verifier to be satisfied, and every
   candidate `high` or `release` task to pass, and rejects incomplete safety
   evidence.
+- **Held-out qualification** — The explicit `praxis eval qualify` surface
+  preflights a pinned provider,
+  profile, and model, then runs the immutable held-out corpus as 36 local
+  Project Eval runs. Baseline-only results use `qualified: null`; candidate
+  claims require complete safe, comparable evidence, and unknown usage/cost
+  keeps optimization claims disabled. A bounded DeepSeek baseline for the
+  exact `anthropic/default/deepseek-v4-flash` pin completed 36/36 runs with
+  33/36 passes (91.7%) and 36/36 safety passes. The
+  `config-kit.add-json-output` run 3 failed closed with provider error
+  `Provider reported max_tokens with completed tool calls`. The
+  `config-kit.preserve-zero-values` run 2 and `task-store.fix-completed-filter`
+  run 3 failed closed with provider error `Provider transport failed`; their
+  terminal results do not establish the transport-error cause. All three were
+  non-safety failures with unknown usage/cost and unsatisfied or not-run
+  behavior verification.
+  Median/p95 turns were 5/7 and median/p95 duration was 28,653.5/52,197 ms.
+  The 33-run known-cost subtotal is USD 0.068322756, not a total. This
+  measures only that pinned 32,768-context/4,096-output configuration;
+  baseline-only evidence remains `qualified: null`, and no live-model quality
+  or optimization claim is established.
 - **Local agent runtime** — C+ Quiet Operator responsive TUI with a linear
   `❯` user / `⏺` assistant conversation, `✻` thinking activity, and `!` shell
   composer grammar, compact stable tool rows, responsive density,
@@ -407,7 +427,7 @@ normal/low-capability full-frame p95 budgets of `<16.7/<33 ms`.
 `npm run test:coverage` measures all production code under `src/**` with V8 and
 enforces global floors of 79% statements, 70% branches, 85% functions, and 81% lines,
 and rejects any production runtime module with zero covered statements (while allowing
-type-only modules). `npm run test:fixtures` executes the 75-behavior native contract; 67 behaviors
+type-only modules). `npm run test:fixtures` executes the 76-behavior native contract; 68 behaviors
 are qualified and 8 are explicitly excluded. Schema-v2 risk tiers and executable evidence dimensions
 are enforced fail-closed. `npm run verify:fixture-contracts` performs the structural check and is part
 of `npm run check`.

package/dist/build-identity.json

@@ -1 +1 @@
-{"schema_version":"1.0","source_revision":"git:ef170d59a802ce59a82fd5cae23c0adaff912d8a","source_dirty":false,"artifact_sha256":"sha256:60ff6f473f030f3a5cd5c2db430f866898607b9f1c205f55ed68abcc2eff1b0d"}
+{"schema_version":"1.0","source_revision":"git:1f78926ebc463fed5e2d7a0f22f49c9569171350","source_dirty":false,"artifact_sha256":"sha256:46543a71fca8edab43c1a8ce3259ec7459bc8e31cdeec7f5028c049d7680350a"}

package/dist/cli-runtime.js

@@ -2336,6 +2336,10 @@ const defaultPluginEvalRuntimeFactory = {
                 tools: [...options.allowedTools],
                 permissionMode: 'dontAsk',
                 ...(options.model ? { model: options.model } : {}),
+                ...(options.provider ? { provider: options.provider } : {}),
+                ...(options.providerProfile
+                    ? { providerProfile: options.providerProfile }
+                    : {}),
                 ...(options.appendSystemPrompt
                     ? { appendSystemPrompt: options.appendSystemPrompt }
                     : {}),
@@ -2368,6 +2372,10 @@ const defaultProjectEvalRuntimeFactory = {
             cwd: options.cwd,
             environment,
             ...(options.model === undefined ? {} : { model: options.model }),
+            ...(options.provider === undefined ? {} : { provider: options.provider }),
+            ...(options.providerProfile === undefined
+                ? {}
+                : { profile: options.providerProfile }),
             includeSettings: true,
             includeProjectSettings: false,
         });
@@ -5116,6 +5124,14 @@ async function execute(argv, io, dependencies, signal) {
             io.stdout(PROJECT_EVAL_COMPARE_HELP);
             return 0;
         }
+        if (special.args[1] === 'qualify' &&
+            special.args
+                .slice(2)
+                .some((value) => value === '-h' || value === '--help')) {
+            const { HELD_OUT_QUALIFICATION_HELP } = await import('./evals/held-out-qualification.js');
+            io.stdout(HELD_OUT_QUALIFICATION_HELP);
+            return 0;
+        }
         if (special.args
             .slice(1)
             .some((value) => value === '-h' || value === '--help')) {
@@ -5124,12 +5140,25 @@ async function execute(argv, io, dependencies, signal) {
         }
         if (!dependencies.projectEval)
             throw new Error('Project eval unavailable');
-        return executeProjectEvalCommand([
+        const prefixFlags = [
             ...(specialPrefix?.model === undefined
                 ? []
                 : ['--model', specialPrefix.model]),
-            ...special.args.slice(1),
-        ], io, dependencies.projectEval, process.cwd(), signal);
+            ...(specialPrefix?.provider === undefined
+                ? []
+                : ['--provider', specialPrefix.provider]),
+            ...(specialPrefix?.providerProfile === undefined
+                ? []
+                : ['--profile', specialPrefix.providerProfile]),
+        ];
+        if (special.args[1] === 'compare' && prefixFlags.length > 0)
+            throw new Error('eval compare does not accept global provider, profile, or model options');
+        const evalArgs = special.args[1] === 'qualify'
+            ? ['qualify', ...prefixFlags, ...special.args.slice(2)]
+            : special.args[1] === 'compare'
+                ? ['compare', ...special.args.slice(2)]
+                : [...prefixFlags, ...special.args.slice(1)];
+        return executeProjectEvalCommand(evalArgs, io, dependencies.projectEval, process.cwd(), signal);
     }
     if ((special.args[0] === 'plugin' || special.args[0] === 'plugins') &&
         special.args[1] === 'eval') {

package/dist/evals/eval-contract.d.ts

@@ -14,6 +14,8 @@ export interface EvalRuntimeFactoryOptions {
     cwd: string;
     configRoot: string;
     home: string;
+    provider?: string;
+    providerProfile?: string;
     model?: string;
     maxTurns: number;
     pluginDirectories?: readonly string[];

package/dist/evals/held-out-qualification.d.ts

@@ -0,0 +1,140 @@
+import { type PraxisBuildIdentity } from '../platform/praxis-build-identity.js';
+import { type ProjectEvalIdentity } from './project-eval-identity.js';
+import { type ProjectEvalComparisonMetric } from './project-eval-comparison.js';
+import type { ProjectEvalDependencies, ProjectEvalRunSummary } from './project-eval.js';
+/**
+ * Qualification orchestration for the immutable held-out corpus.
+ *
+ * This module owns the qualification contract and evidence envelope; actual
+ * case execution remains in the Project Eval runner.
+ */
+export declare const HELD_OUT_QUALIFICATION_HELP = "Usage: praxis eval qualify [options] <corpus>\n\nQualify an explicitly pinned provider/model against the immutable held-out corpus.\n\nOptions:\n\nRequired:\n  --provider <id>             Provider identifier\n  --profile <id>              Provider profile identifier\n  --model <id>                Model identifier\n  --confirm-held-out <id@sha256>  Confirm the exact held-out corpus digest\n  --run-verification          Run the declared case verifiers\n  --output-dir <dir>          Write qualification artifacts here\n  --allow-tools <rules>       Grant gated tools; comma-separated and repeatable\n\nOptional:\n  --baseline <qualification-result.json>  Compare against a completed result\n  --keep-temp                 Preserve temporary workspaces\n  --json                      Print exactly one qualification JSON value\n  --verbose                   Print run progress to stderr\n  -h, --help                  Display help";
+export interface HeldOutQualificationOptions {
+    corpus?: string;
+    provider?: string;
+    profile?: string;
+    model?: string;
+    confirmHeldOut?: string;
+    allowTools: string[];
+    runVerification: boolean;
+    outputDir?: string;
+    baseline?: string;
+    keepTemp: boolean;
+    json: boolean;
+    verbose: boolean;
+    help?: true;
+}
+/** Parse the strict, side-effect-free qualification command line. */
+export declare function parseHeldOutQualificationOptions(argv: readonly string[]): HeldOutQualificationOptions;
+export interface HeldOutQualificationAggregateReference {
+    repository: string;
+    path: string;
+    sha256: `sha256:${string}`;
+    identity_sha256: `sha256:${string}`;
+}
+export interface HeldOutQualificationRun {
+    repository: string;
+    case: string;
+    run: number;
+    passed: boolean;
+    safety_passed: boolean;
+    verifier_satisfied: boolean;
+    turns: number;
+    duration_ms: number;
+    usage_known: boolean;
+    cost_known: boolean;
+    usage: ProjectEvalRunSummary['usage'];
+    cost_usd: number | null;
+    identity: ProjectEvalIdentity;
+}
+export interface HeldOutQualificationCaseSummary {
+    repository: string;
+    case: string;
+    repetitions: number;
+    passed: number;
+    failed: number;
+    safety_passed: number;
+    safety_failed: number;
+    verifier_satisfied_runs: number;
+    verifier_unsatisfied_runs: number;
+    passed_all: boolean;
+    safety_passed_all: boolean;
+    verifier_satisfied: boolean;
+}
+export interface HeldOutQualificationResult {
+    schema_version: '1.0';
+    corpus: {
+        id: string;
+        version: number;
+        content_sha256: `sha256:${string}`;
+        repository_count: number;
+        task_count: number;
+        repetitions: number;
+        planned_run_count: number;
+    };
+    provider: string;
+    profile: string;
+    protocol: string;
+    model: string;
+    endpoint_sha256: `sha256:${string}`;
+    plan_sha256: `sha256:${string}`;
+    praxis_version: string;
+    build: PraxisBuildIdentity;
+    node_version: string;
+    platform: string;
+    architecture: string;
+    start: string;
+    duration_ms: number;
+    aggregates: readonly HeldOutQualificationAggregateReference[];
+    planned_run_count: number;
+    completed_run_count: number;
+    passed: number;
+    failed: number;
+    safety_passed: number;
+    safety_failed: number;
+    verifier_satisfied_runs: number;
+    verifier_unsatisfied_runs: number;
+    usage_known_runs: number;
+    usage_unknown_runs: number;
+    cost_known_runs: number;
+    cost_unknown_runs: number;
+    usage_totals: {
+        input_tokens: number;
+        output_tokens: number;
+        cache_read_input_tokens: number;
+        cache_creation_input_tokens: number;
+        web_search_requests: number;
+    } | null;
+    known_cost_total_usd: number | null;
+    median_turns: number;
+    p95_turns: number;
+    median_duration_ms: number;
+    p95_duration_ms: number;
+    cases: readonly HeldOutQualificationCaseSummary[];
+    runs: readonly HeldOutQualificationRun[];
+    baseline?: {
+        source_sha256: `sha256:${string}`;
+        summary: {
+            passed: number;
+            failed: number;
+            pass_rate: number;
+            safety_pass_rate: number;
+            planned_run_count: number;
+        };
+    };
+    regressions: readonly {
+        case: string;
+        run: number;
+        baseline_passed: boolean;
+        candidate_passed: boolean;
+    }[];
+    metric_deltas: Readonly<Record<string, ProjectEvalComparisonMetric<number | null>>>;
+    qualified: boolean | null;
+    optimization_claim_allowed: boolean;
+}
+export type HeldOutQualificationDependencies = ProjectEvalDependencies;
+export declare function executeHeldOutQualificationCommand(argv: readonly string[], io: {
+    stdout(message: string): void;
+    stderr(message: string): void;
+}, dependencies: HeldOutQualificationDependencies, callerCwd?: string, signal?: AbortSignal): Promise<number>;
+//# sourceMappingURL=held-out-qualification.d.ts.map