evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/reason_codes.py

@@ -0,0 +1,103 @@
+"""Standardized failure reason codes (T10).
+
+Port of the TypeScript SDK's ``reason-codes.ts``.
+"""
+
+from __future__ import annotations
+
+from typing import Literal
+
+ReasonCode = Literal[
+    "PASS",
+    "WARN_REGRESSION",
+    "LOW_SAMPLE_SIZE",
+    "BASELINE_MISSING",
+    "SCORE_TOO_LOW",
+    "DELTA_TOO_HIGH",
+    "COST_BUDGET_EXCEEDED",
+    "LATENCY_BUDGET_EXCEEDED",
+    "POLICY_FAILED",
+    "UNKNOWN",
+    # Legacy aliases
+    "LOW_SCORE",
+    "LOW_PASS_RATE",
+    "SAFETY_RISK",
+    "LATENCY_RISK",
+    "COST_RISK",
+    "MAX_DROP_EXCEEDED",
+    "INSUFFICIENT_EVIDENCE",
+    "POLICY_VIOLATION",
+]
+
+REASON_CODES: dict[str, dict[str, str]] = {
+    "PASS": {"label": "Pass", "severity": "info", "description": "All checks passed."},
+    "WARN_REGRESSION": {
+        "label": "Warning: Regression",
+        "severity": "warn",
+        "description": "Score dropped but within tolerance.",
+    },
+    "LOW_SAMPLE_SIZE": {
+        "label": "Low Sample Size",
+        "severity": "warn",
+        "description": "Not enough data points for confidence.",
+    },
+    "BASELINE_MISSING": {
+        "label": "Baseline Missing",
+        "severity": "warn",
+        "description": "No baseline to compare against.",
+    },
+    "SCORE_TOO_LOW": {
+        "label": "Score Too Low",
+        "severity": "fail",
+        "description": "Score is below the minimum threshold.",
+    },
+    "DELTA_TOO_HIGH": {
+        "label": "Delta Too High",
+        "severity": "fail",
+        "description": "Score dropped more than the allowed delta.",
+    },
+    "COST_BUDGET_EXCEEDED": {
+        "label": "Cost Budget Exceeded",
+        "severity": "fail",
+        "description": "Evaluation cost exceeded the budget.",
+    },
+    "LATENCY_BUDGET_EXCEEDED": {
+        "label": "Latency Budget Exceeded",
+        "severity": "fail",
+        "description": "Response latency exceeded the limit.",
+    },
+    "POLICY_FAILED": {"label": "Policy Failed", "severity": "fail", "description": "One or more policy checks failed."},
+    "UNKNOWN": {"label": "Unknown", "severity": "fail", "description": "Unknown failure reason."},
+    # Legacy aliases
+    "LOW_SCORE": {"label": "Low Score", "severity": "fail", "description": "Score is below the minimum threshold."},
+    "LOW_PASS_RATE": {
+        "label": "Low Pass Rate",
+        "severity": "fail",
+        "description": "Pass rate is below the minimum threshold.",
+    },
+    "SAFETY_RISK": {"label": "Safety Risk", "severity": "fail", "description": "Safety check failed."},
+    "LATENCY_RISK": {"label": "Latency Risk", "severity": "warn", "description": "Latency is near or above the limit."},
+    "COST_RISK": {"label": "Cost Risk", "severity": "warn", "description": "Cost is near or above the budget."},
+    "MAX_DROP_EXCEEDED": {
+        "label": "Max Drop Exceeded",
+        "severity": "fail",
+        "description": "Score dropped more than the allowed maximum.",
+    },
+    "INSUFFICIENT_EVIDENCE": {
+        "label": "Insufficient Evidence",
+        "severity": "warn",
+        "description": "Not enough data to make a determination.",
+    },
+    "POLICY_VIOLATION": {"label": "Policy Violation", "severity": "fail", "description": "A policy was violated."},
+}
+
+
+def get_reason_info(code: str) -> dict[str, str]:
+    """Get label, severity, and description for a reason code."""
+    return REASON_CODES.get(code, REASON_CODES["UNKNOWN"])
+
+
+def is_blocking(code: str) -> bool:
+    """Return True if the reason code should block a deployment."""
+    info = get_reason_info(code)
+    return info["severity"] == "fail"

evalgate_sdk/regression.py

@@ -0,0 +1,196 @@
+"""Regression gate constants, types, and helpers."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+class GATE_EXIT:
+    PASS = 0
+    REGRESSION = 1
+    INFRA_ERROR = 2
+    CONFIDENCE_FAILED = 3
+    CONFIDENCE_MISSING = 4
+
+    def __class_getitem__(cls, key: str) -> int:
+        """Support dict-style access: GATE_EXIT['PASS'] == 0."""
+        return getattr(cls, key)
+
+
+class GATE_CATEGORY:
+    PASS = "pass"
+    REGRESSION = "regression"
+    INFRA_ERROR = "infra_error"
+
+
+REPORT_SCHEMA_VERSION = 1
+
+
+class ARTIFACTS:
+    BASELINE = "evals/baseline.json"
+    REGRESSION_REPORT = "evals/regression-report.json"
+    CONFIDENCE_SUMMARY = "evals/confidence-summary.json"
+    LATENCY_BENCHMARK = "evals/latency-benchmark.json"
+
+
+@dataclass
+class BaselineTolerance:
+    score_drop: float = 0.05
+    latency_increase_pct: float = 20.0
+    min_confidence: float = 0.8
+
+
+@dataclass
+class Baseline:
+    version: int = REPORT_SCHEMA_VERSION
+    scores: dict[str, float] = field(default_factory=dict)
+    latencies: dict[str, float] = field(default_factory=dict)
+    created_at: str | None = None
+    tolerance: BaselineTolerance = field(default_factory=BaselineTolerance)
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+
+@dataclass
+class RegressionDelta:
+    test_id: str
+    metric: str
+    baseline_value: float
+    current_value: float
+    delta: float
+    delta_pct: float
+    category: str = GATE_CATEGORY.PASS
+    severity: Literal["low", "medium", "high", "critical"] = "low"
+
+
+@dataclass
+class RegressionReport:
+    version: int = REPORT_SCHEMA_VERSION
+    run_id: str = ""
+    gate_exit: int = GATE_EXIT.PASS
+    gate_category: str = GATE_CATEGORY.PASS
+    deltas: list[RegressionDelta] = field(default_factory=list)
+    summary: dict[str, Any] = field(default_factory=dict)
+    created_at: str | None = None
+
+
+def evaluate_regression(
+    baseline: Baseline,
+    current_scores: dict[str, float],
+    *,
+    min_score: float | None = None,
+    max_drop: float | None = None,
+) -> RegressionReport:
+    """Compare current scores against a baseline and produce a regression report.
+
+    Args:
+        baseline: Baseline scores and tolerance configuration.
+        current_scores: Current run scores keyed by test ID.
+        min_score: Optional absolute floor — any score below this is a failure.
+        max_drop: Optional override for baseline.tolerance.score_drop.
+    """
+    effective_drop = max_drop if max_drop is not None else baseline.tolerance.score_drop
+
+    deltas: list[RegressionDelta] = []
+    gate_exit = GATE_EXIT.PASS
+    gate_category = GATE_CATEGORY.PASS
+
+    for test_id, baseline_score in baseline.scores.items():
+        current = current_scores.get(test_id)
+        if current is None:
+            continue
+        delta = current - baseline_score
+        delta_pct = (delta / baseline_score * 100) if baseline_score != 0 else 0
+
+        failed = False
+
+        if delta < -effective_drop:
+            failed = True
+
+        if min_score is not None and current < min_score:
+            failed = True
+
+        if failed:
+            severity: Literal["low", "medium", "high", "critical"] = "high" if abs(delta_pct) > 20 else "medium"
+            deltas.append(
+                RegressionDelta(
+                    test_id=test_id,
+                    metric="score",
+                    baseline_value=baseline_score,
+                    current_value=current,
+                    delta=delta,
+                    delta_pct=delta_pct,
+                    category=GATE_CATEGORY.REGRESSION,
+                    severity=severity,
+                )
+            )
+            gate_exit = GATE_EXIT.REGRESSION
+            gate_category = GATE_CATEGORY.REGRESSION
+        else:
+            deltas.append(
+                RegressionDelta(
+                    test_id=test_id,
+                    metric="score",
+                    baseline_value=baseline_score,
+                    current_value=current,
+                    delta=delta,
+                    delta_pct=delta_pct,
+                )
+            )
+
+    # Check min_score for tests not in the baseline
+    if min_score is not None:
+        for test_id, current in current_scores.items():
+            if test_id in baseline.scores:
+                continue
+            if current < min_score:
+                deltas.append(
+                    RegressionDelta(
+                        test_id=test_id,
+                        metric="score",
+                        baseline_value=0.0,
+                        current_value=current,
+                        delta=current,
+                        delta_pct=0.0,
+                        category=GATE_CATEGORY.REGRESSION,
+                        severity="medium",
+                    )
+                )
+                gate_exit = GATE_EXIT.REGRESSION
+                gate_category = GATE_CATEGORY.REGRESSION
+
+    return RegressionReport(
+        gate_exit=gate_exit,
+        gate_category=gate_category,
+        deltas=deltas,
+        summary={
+            "total": len(deltas),
+            "regressions": sum(1 for d in deltas if d.category == GATE_CATEGORY.REGRESSION),
+            "passed": sum(1 for d in deltas if d.category == GATE_CATEGORY.PASS),
+        },
+    )
+
+
+def compute_baseline_checksum(baseline: Baseline) -> str:
+    """Compute a deterministic SHA-256 checksum of a baseline's score data.
+
+    The checksum covers ``version``, ``scores``, and ``latencies`` so that
+    any tamper with the stored values is detectable.
+    """
+    payload = json.dumps(
+        {
+            "version": baseline.version,
+            "scores": dict(sorted(baseline.scores.items())),
+            "latencies": dict(sorted(baseline.latencies.items())),
+        },
+        sort_keys=True,
+        separators=(",", ":"),
+    )
+    return hashlib.sha256(payload.encode("utf-8")).hexdigest()
+
+
+def verify_baseline_checksum(baseline: Baseline, expected_checksum: str) -> bool:
+    """Return ``True`` if the baseline's current checksum matches *expected_checksum*."""
+    return compute_baseline_checksum(baseline) == expected_checksum

evalgate_sdk/replay_decision.py

@@ -0,0 +1,115 @@
+from __future__ import annotations
+
+from dataclasses import dataclass
+from typing import Literal
+
+from evalgate_sdk.golden import RunMetrics, extract_run_metrics
+
+
+@dataclass(slots=True)
+class NormalizedBudgetConfig:
+    mode: Literal["traces", "cost"]
+    max_traces: int | None = None
+    max_cost_usd: float | None = None
+
+
+@dataclass(slots=True)
+class ReplayDecision:
+    action: Literal["keep", "discard"]
+    reason: Literal["pass_rate_improved", "pass_rate_declined", "budget_exceeded"]
+    previous_pass_rate: float
+    new_pass_rate: float
+    previous_corrected_pass_rate: float | None
+    new_corrected_pass_rate: float | None
+    comparison_basis: Literal["corrected", "raw"]
+    budget_used: float
+    budget_limit: float
+
+    def to_dict(self) -> dict[str, float | str | None]:
+        return {
+            "action": self.action,
+            "reason": self.reason,
+            "previousPassRate": self.previous_pass_rate,
+            "newPassRate": self.new_pass_rate,
+            "previousCorrectedPassRate": self.previous_corrected_pass_rate,
+            "newCorrectedPassRate": self.new_corrected_pass_rate,
+            "comparisonBasis": self.comparison_basis,
+            "budgetUsed": self.budget_used,
+            "budgetLimit": self.budget_limit,
+        }
+
+
+def determine_comparison_basis(
+    previous_corrected: float | None,
+    new_corrected: float | None,
+) -> Literal["corrected", "raw"]:
+    if previous_corrected is not None and new_corrected is not None:
+        return "corrected"
+    return "raw"
+
+
+def _budget_values(metrics: RunMetrics, budget: NormalizedBudgetConfig) -> tuple[float, float]:
+    if budget.mode == "traces":
+        used = float(metrics.total_results)
+        limit = float(budget.max_traces or 0)
+        return used, limit
+    used = float(metrics.total_cost_usd or 0.0)
+    limit = float(budget.max_cost_usd or 0.0)
+    return used, limit
+
+
+def evaluate_replay_outcome(
+    previous_run: dict,
+    new_run: dict,
+    budget_config: NormalizedBudgetConfig,
+) -> ReplayDecision:
+    previous_metrics = extract_run_metrics(previous_run)
+    new_metrics = extract_run_metrics(new_run)
+    budget_used, budget_limit = _budget_values(new_metrics, budget_config)
+    comparison_basis = determine_comparison_basis(
+        previous_metrics.corrected_pass_rate_ratio,
+        new_metrics.corrected_pass_rate_ratio,
+    )
+    if budget_used > budget_limit:
+        return ReplayDecision(
+            action="discard",
+            reason="budget_exceeded",
+            previous_pass_rate=previous_metrics.pass_rate_ratio,
+            new_pass_rate=new_metrics.pass_rate_ratio,
+            previous_corrected_pass_rate=previous_metrics.corrected_pass_rate_ratio,
+            new_corrected_pass_rate=new_metrics.corrected_pass_rate_ratio,
+            comparison_basis=comparison_basis,
+            budget_used=budget_used,
+            budget_limit=budget_limit,
+        )
+
+    previous_rate = (
+        previous_metrics.corrected_pass_rate_ratio if comparison_basis == "corrected" else previous_metrics.pass_rate_ratio
+    )
+    new_rate = new_metrics.corrected_pass_rate_ratio if comparison_basis == "corrected" else new_metrics.pass_rate_ratio
+    previous_rate = previous_rate if previous_rate is not None else previous_metrics.pass_rate_ratio
+    new_rate = new_rate if new_rate is not None else new_metrics.pass_rate_ratio
+
+    if new_rate > previous_rate:
+        return ReplayDecision(
+            action="keep",
+            reason="pass_rate_improved",
+            previous_pass_rate=previous_metrics.pass_rate_ratio,
+            new_pass_rate=new_metrics.pass_rate_ratio,
+            previous_corrected_pass_rate=previous_metrics.corrected_pass_rate_ratio,
+            new_corrected_pass_rate=new_metrics.corrected_pass_rate_ratio,
+            comparison_basis=comparison_basis,
+            budget_used=budget_used,
+            budget_limit=budget_limit,
+        )
+    return ReplayDecision(
+        action="discard",
+        reason="pass_rate_declined",
+        previous_pass_rate=previous_metrics.pass_rate_ratio,
+        new_pass_rate=new_metrics.pass_rate_ratio,
+        previous_corrected_pass_rate=previous_metrics.corrected_pass_rate_ratio,
+        new_corrected_pass_rate=new_metrics.corrected_pass_rate_ratio,
+        comparison_basis=comparison_basis,
+        budget_used=budget_used,
+        budget_limit=budget_limit,
+    )

evalgate_sdk/runtime/__init__.py

@@ -0,0 +1,50 @@
+"""Runtime foundation — defineEval DSL, registry, executor."""
+
+from evalgate_sdk.runtime.eval import create_result, define_eval, define_suite
+from evalgate_sdk.runtime.executor import create_local_executor, default_local_executor
+from evalgate_sdk.runtime.registry import (
+    create_eval_runtime,
+    dispose_active_runtime,
+    get_active_runtime,
+    set_active_runtime,
+    with_runtime,
+)
+from evalgate_sdk.runtime.types import (
+    EvalContext,
+    EvalExecutionError,
+    EvalResult,
+    EvalRuntimeError,
+    EvalSpec,
+    ExecutorCapabilities,
+    SpecConfig,
+    SpecExecutionError,
+    SpecOptions,
+    SpecRegistrationError,
+)
+from evalgate_sdk.runtime.types import (
+    RuntimeError as EvalSDKRuntimeError,
+)
+
+__all__ = [
+    "define_eval",
+    "define_suite",
+    "create_result",
+    "create_local_executor",
+    "default_local_executor",
+    "create_eval_runtime",
+    "get_active_runtime",
+    "set_active_runtime",
+    "dispose_active_runtime",
+    "with_runtime",
+    "EvalSpec",
+    "EvalContext",
+    "EvalResult",
+    "SpecConfig",
+    "SpecOptions",
+    "ExecutorCapabilities",
+    "EvalRuntimeError",
+    "SpecRegistrationError",
+    "SpecExecutionError",
+    "EvalSDKRuntimeError",
+    "EvalExecutionError",
+]

evalgate_sdk/runtime/adapters/__init__.py

@@ -0,0 +1 @@
+"""Runtime adapters for legacy → DSL migration."""