evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/cli/regression_gate.py

@@ -0,0 +1,312 @@
+"""evalgate gate — Run the regression gate.
+
+Two modes:
+  1. Project mode: delegates to eval:regression-gate script (full gate)
+  2. Built-in mode: runs tests, compares against baseline
+
+Port of ``cli/regression-gate.ts``.
+"""
+
+from __future__ import annotations
+
+import json
+import os
+import re
+import subprocess
+import time
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+from typing import Any, Literal
+
+
+@dataclass
+class GateArgs:
+    format: Literal["human", "json", "github"] = "human"
+
+
+@dataclass
+class BuiltinReport:
+    schema_version: int = 1
+    timestamp: str = ""
+    exit_code: int = 0
+    category: str = "pass"
+    passed: bool = True
+    failures: list[str] = field(default_factory=list)
+    deltas: list[dict[str, Any]] = field(default_factory=list)
+    baseline: dict[str, str] | None = None
+    duration_ms: int = 0
+    command: str = ""
+    runner: str = ""
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "schemaVersion": self.schema_version,
+            "timestamp": self.timestamp,
+            "exitCode": self.exit_code,
+            "category": self.category,
+            "passed": self.passed,
+            "failures": self.failures,
+            "deltas": self.deltas,
+            "baseline": self.baseline,
+            "durationMs": self.duration_ms,
+            "command": self.command,
+            "runner": self.runner,
+        }
+
+
+def parse_gate_args(argv: list[str]) -> GateArgs:
+    args = GateArgs()
+    i = 0
+    while i < len(argv):
+        if argv[i] == "--format" and i + 1 < len(argv):
+            fmt = argv[i + 1]
+            if fmt in ("human", "json", "github"):
+                args.format = fmt  # type: ignore[assignment]
+            i += 2
+        else:
+            i += 1
+    return args
+
+
+def _detect_test_runner(cwd: str) -> str:
+    """Detect Python test runner used in the project."""
+    pyproject = os.path.join(cwd, "pyproject.toml")
+    if os.path.isfile(pyproject):
+        try:
+            text = Path(pyproject).read_text(encoding="utf-8")
+            if "pytest" in text:
+                return "pytest"
+            if "unittest" in text:
+                return "unittest"
+        except OSError:
+            pass
+
+    if os.path.isfile(os.path.join(cwd, "pytest.ini")) or os.path.isfile(os.path.join(cwd, "setup.cfg")):
+        return "pytest"
+
+    return "pytest"
+
+
+def _detect_test_command(cwd: str) -> str:
+    """Detect the test command to run."""
+    runner = _detect_test_runner(cwd)
+    if runner == "pytest":
+        return "python -m pytest"
+    return "python -m unittest discover"
+
+
+def run_builtin_gate(cwd: str) -> BuiltinReport:
+    """Run the built-in lightweight gate."""
+    t0 = time.time()
+    now = datetime.now(timezone.utc).isoformat()
+    command = _detect_test_command(cwd)
+    runner = _detect_test_runner(cwd)
+    baseline_path = os.path.join(cwd, "evals", "baseline.json")
+
+    if not os.path.isfile(baseline_path):
+        return BuiltinReport(
+            timestamp=now,
+            exit_code=2,
+            category="infra_error",
+            passed=False,
+            failures=["Baseline file not found. Run: evalgate init"],
+            duration_ms=int((time.time() - t0) * 1000),
+            command=command,
+            runner=runner,
+        )
+
+    try:
+        baseline_data = json.loads(Path(baseline_path).read_text(encoding="utf-8"))
+    except (OSError, json.JSONDecodeError):
+        return BuiltinReport(
+            timestamp=now,
+            exit_code=2,
+            category="infra_error",
+            passed=False,
+            failures=["Failed to parse evals/baseline.json"],
+            duration_ms=int((time.time() - t0) * 1000),
+            command=command,
+            runner=runner,
+        )
+
+    baseline_meta = None
+    if baseline_data.get("updatedAt"):
+        baseline_meta = {
+            "updatedAt": baseline_data["updatedAt"],
+            "updatedBy": baseline_data.get("updatedBy", "unknown"),
+        }
+
+    # Run tests
+    try:
+        result = subprocess.run(
+            command.split(),
+            cwd=cwd,
+            capture_output=True,
+            text=True,
+            timeout=300,
+        )
+    except subprocess.TimeoutExpired:
+        return BuiltinReport(
+            timestamp=now,
+            exit_code=2,
+            category="infra_error",
+            passed=False,
+            failures=["Test command timed out after 300s"],
+            duration_ms=int((time.time() - t0) * 1000),
+            command=command,
+            runner=runner,
+        )
+    except OSError as exc:
+        return BuiltinReport(
+            timestamp=now,
+            exit_code=2,
+            category="infra_error",
+            passed=False,
+            failures=[f"Failed to run test command: {exc}"],
+            duration_ms=int((time.time() - t0) * 1000),
+            command=command,
+            runner=runner,
+        )
+
+    tests_passed = result.returncode == 0
+    output = (result.stdout or "") + (result.stderr or "")
+
+    # Extract test count
+    test_count = 0
+    count_match = (
+        re.search(r"(\d+)\s+(?:tests?|specs?)\s+(?:passed|completed)", output, re.I)
+        or re.search(r"(\d+)\s+passed", output, re.I)
+        or re.search(r"(\d+)\s+passing", output, re.I)
+    )
+    if count_match:
+        test_count = int(count_match.group(1))
+
+    baseline_passed = baseline_data.get("confidenceTests", {}).get("passed", True)
+    baseline_total = baseline_data.get("confidenceTests", {}).get("total", 0)
+
+    failures: list[str] = []
+    deltas: list[dict[str, Any]] = []
+
+    deltas.append(
+        {
+            "metric": "tests_passing",
+            "baseline": baseline_passed,
+            "current": tests_passed,
+            "delta": "0" if tests_passed == baseline_passed else ("+1" if tests_passed else "-1"),
+            "status": "pass" if tests_passed else "fail",
+        }
+    )
+
+    if not tests_passed and baseline_passed:
+        failures.append("Tests were passing in baseline but are now failing")
+
+    if test_count > 0 or baseline_total > 0:
+        count_delta = test_count - baseline_total
+        deltas.append(
+            {
+                "metric": "test_count",
+                "baseline": baseline_total,
+                "current": test_count,
+                "delta": f"+{count_delta}" if count_delta >= 0 else str(count_delta),
+                "status": "pass" if test_count >= baseline_total else "fail",
+            }
+        )
+        if test_count < baseline_total:
+            failures.append(f"Test count dropped from {baseline_total} to {test_count} ({count_delta})")
+
+    has_regression = len(failures) > 0
+
+    return BuiltinReport(
+        timestamp=now,
+        exit_code=1 if has_regression else 0,
+        category="regression" if has_regression else "pass",
+        passed=not has_regression,
+        failures=failures,
+        deltas=deltas,
+        baseline=baseline_meta,
+        duration_ms=int((time.time() - t0) * 1000),
+        command=command,
+        runner=runner,
+    )
+
+
+def format_human(report: BuiltinReport) -> str:
+    """Format report for human consumption."""
+    icon = "✅" if report.passed else "❌"
+    lines = [f"\n{icon} EvalGate Gate: {report.category.upper()}\n"]
+
+    if report.deltas:
+
+        def pad(s, n):
+            return str(s).ljust(n)
+
+        lines.append(f"  {pad('Metric', 16)} {pad('Baseline', 10)} {pad('Current', 10)} {pad('Delta', 8)} Status")
+        lines.append(f"  {'-' * 16} {'-' * 10} {'-' * 10} {'-' * 8} ------")
+        for d in report.deltas:
+            si = "✔" if d["status"] == "pass" else "✖"
+            lines.append(
+                f"  {pad(d['metric'], 16)} {pad(d['baseline'], 10)} {pad(d['current'], 10)} {pad(d['delta'], 8)} {si}"
+            )
+
+    if report.failures:
+        lines.append("\n  Failures:")
+        for f in report.failures:
+            lines.append(f"    • {f}")
+    lines.append("")
+
+    return "\n".join(lines)
+
+
+def format_github(report: BuiltinReport) -> str:
+    """Format report as GitHub markdown."""
+    icon = "✅" if report.passed else "❌"
+    lines = [
+        f"## {icon} EvalGate Gate: {report.category}",
+        "",
+        "| Metric | Baseline | Current | Delta | Status |",
+        "|--------|----------|---------|-------|--------|",
+    ]
+    for d in report.deltas:
+        si = "✅" if d["status"] == "pass" else "❌"
+        lines.append(f"| {d['metric']} | {d['baseline']} | {d['current']} | {d['delta']} | {si} |")
+
+    if report.failures:
+        lines.extend(["", "### Failures", ""])
+        for f in report.failures:
+            lines.append(f"- {f}")
+
+    lines.append(f"\nSchema version: {report.schema_version}")
+    return "\n".join(lines)
+
+
+def run_gate(argv: list[str] | None = None) -> int:
+    """Main gate entry point. Returns exit code."""
+    cwd = os.getcwd()
+    args = parse_gate_args(argv or [])
+    report = run_builtin_gate(cwd)
+
+    # Write report artifact
+    evals_dir = os.path.join(cwd, "evals")
+    os.makedirs(evals_dir, exist_ok=True)
+    Path(os.path.join(cwd, "evals", "regression-report.json")).write_text(
+        json.dumps(report.to_dict(), indent=2) + "\n",
+        encoding="utf-8",
+    )
+
+    if args.format == "json":
+        print(json.dumps(report.to_dict(), indent=2))
+    elif args.format == "github":
+        md = format_github(report)
+        summary_path = os.environ.get("GITHUB_STEP_SUMMARY")
+        if summary_path:
+            try:
+                with open(summary_path, "a") as f:
+                    f.write(md + "\n")
+            except OSError:
+                pass
+        print(md)
+    else:
+        print(format_human(report))
+
+    return report.exit_code

evalgate_sdk/cli/render/__init__.py

@@ -0,0 +1 @@
+"""Render utilities for CLI output."""

evalgate_sdk/cli/render/snippet.py

@@ -0,0 +1,18 @@
+"""Truncate a string for deterministic output.
+
+Port of ``cli/render/snippet.ts``.
+"""
+
+from __future__ import annotations
+
+import re
+
+
+def truncate_snippet(s: str | None, max_len: int = 140) -> str:
+    """Replaces newlines with space, caps length."""
+    if s is None:
+        return ""
+    normalized = re.sub(r"\s+", " ", s).strip()
+    if len(normalized) <= max_len:
+        return normalized
+    return normalized[:max_len] + "…"

evalgate_sdk/cli/render/sort.py

@@ -0,0 +1,29 @@
+"""Deterministic ordering for failed cases.
+
+Sort by status severity (failed > error > skipped > passed), then by test_case_id asc.
+
+Port of ``cli/render/sort.ts``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+STATUS_SEVERITY: dict[str, int] = {
+    "failed": 0,
+    "error": 1,
+    "skipped": 2,
+    "passed": 3,
+}
+
+
+def sort_failed_cases(cases: list[dict[str, Any]]) -> list[dict[str, Any]]:
+    """Sort cases by status severity then test_case_id."""
+
+    def sort_key(c: dict[str, Any]) -> tuple[int, int]:
+        status = (c.get("status") or "").lower()
+        sev = STATUS_SEVERITY.get(status, 4)
+        tid = c.get("test_case_id") or c.get("testCaseId") or 0
+        return (sev, tid)
+
+    return sorted(cases, key=sort_key)

evalgate_sdk/cli/report/__init__.py

@@ -0,0 +1 @@
+"""Report building utilities for CLI."""

evalgate_sdk/cli/report/build_check_report.py

@@ -0,0 +1,209 @@
+"""Build CheckReport from API data and gate result.
+
+Normalizes failed cases (truncate, sort), dashboard URL, top N + more.
+
+Port of ``cli/report/build-check-report.ts``.
+"""
+
+from __future__ import annotations
+
+from typing import Any
+
+from evalgate_sdk.cli.formatters.types import (
+    CHECK_REPORT_SCHEMA_VERSION,
+    CheckReport,
+    FailedCase,
+    GateThresholds,
+    ScoreBreakdown01,
+    ScoreContribPts,
+)
+from evalgate_sdk.cli.render.snippet import truncate_snippet
+from evalgate_sdk.cli.render.sort import sort_failed_cases
+
+TOP_N = 3
+SNIPPET_MAX = 50
+
+
+def compute_contrib_pts(b: ScoreBreakdown01) -> ScoreContribPts:
+    """ContribPts from weights: passRate*50, safety*25, (0.6*judge+0.4*schema)*15, (0.6*latency+0.4*cost)*10."""
+    pr = b.pass_rate or 0
+    s = b.safety or 0
+    j = b.judge or 0
+    sc = b.schema or 0
+    lat = b.latency or 0
+    c = b.cost or 0
+    return ScoreContribPts(
+        pass_rate_pts=round(pr * 50 * 10) / 10,
+        safety_pts=round(s * 25 * 10) / 10,
+        compliance_pts=round((0.6 * j + 0.4 * sc) * 15 * 10) / 10,
+        performance_pts=round((0.6 * lat + 0.4 * c) * 10 * 10) / 10,
+    )
+
+
+def build_check_report(
+    evaluation_id: str,
+    quality: dict[str, Any],
+    gate_result: dict[str, Any],
+    base_url: str = "",
+    run_details: dict[str, Any] | None = None,
+    request_id: str | None = None,
+    share_url: str | None = None,
+    baseline_run_id: int | None = None,
+    ci_run_url: str | None = None,
+    explain: bool = False,
+    policy: str | None = None,
+    min_score: float | None = None,
+    max_drop: float | None = None,
+    warn_drop: float | None = None,
+    min_n: int | None = None,
+    allow_weak_evidence: bool | None = None,
+    baseline: str | None = None,
+    max_cost_usd: float | None = None,
+    max_latency_ms: float | None = None,
+    max_cost_delta_usd: float | None = None,
+) -> CheckReport:
+    """Build a CheckReport from API data and gate result."""
+    score = quality.get("score", 0)
+    total = quality.get("total")
+    baseline_score = quality.get("baselineScore") or quality.get("baseline_score")
+    regression_delta = quality.get("regressionDelta") or quality.get("regression_delta")
+    evaluation_run_id = quality.get("evaluationRunId") or quality.get("evaluation_run_id")
+    breakdown = quality.get("breakdown", {})
+    flags = sorted(quality.get("flags", []))
+
+    dashboard_url = None
+    if evaluation_run_id is not None:
+        clean_base = base_url.rstrip("/")
+        dashboard_url = f"{clean_base}/evaluations/{evaluation_id}/runs/{evaluation_run_id}"
+
+    # Build failed cases from run details
+    failed_cases: list[FailedCase] = []
+    if run_details and run_details.get("results") and evaluation_run_id is not None:
+        raw = []
+        for r in run_details["results"]:
+            if r.get("status") == "failed":
+                tc = r.get("test_cases", {})
+                raw.append(
+                    {
+                        "test_case_id": r.get("testCaseId") or r.get("test_case_id"),
+                        "status": "failed",
+                        "name": tc.get("name"),
+                        "input": tc.get("input"),
+                        "expected_output": tc.get("expectedOutput") or tc.get("expected_output"),
+                        "output": r.get("output"),
+                    }
+                )
+
+        sorted_raw = sort_failed_cases(raw)
+        for fc in sorted_raw:
+            failed_cases.append(
+                FailedCase(
+                    test_case_id=fc.get("test_case_id"),
+                    status="failed",
+                    name=fc.get("name"),
+                    input=fc.get("input"),
+                    input_snippet=truncate_snippet(fc.get("input"), SNIPPET_MAX),
+                    expected_output=fc.get("expected_output"),
+                    expected_snippet=truncate_snippet(fc.get("expected_output"), SNIPPET_MAX),
+                    output=fc.get("output"),
+                    output_snippet=truncate_snippet(fc.get("output"), SNIPPET_MAX),
+                )
+            )
+
+    failed_cases_shown = min(len(failed_cases), TOP_N) if failed_cases else None
+    failed_cases_more = (len(failed_cases) - TOP_N) if len(failed_cases) > TOP_N else None
+
+    gate_skipped = gate_result.get("gate_skipped", gate_result.get("gateSkipped", False))
+    gate_applied = not gate_skipped
+    gate_mode = "neutral" if gate_skipped else "enforced"
+    reason_code = gate_result.get("reason_code", gate_result.get("reasonCode", "UNKNOWN"))
+
+    if reason_code == "WARN_REGRESSION":
+        verdict = "warn"
+    elif gate_result.get("passed"):
+        verdict = "pass"
+    else:
+        verdict = "fail"
+
+    actionable_message = None
+    if gate_skipped:
+        actionable_message = (
+            "Gate not applied: baseline missing. Publish a baseline from the dashboard, "
+            "or run with --baseline previous once you have runs."
+        )
+    else:
+        actionable_message = gate_result.get("reason_message") or gate_result.get("reasonMessage")
+
+    breakdown_01 = None
+    if breakdown:
+        breakdown_01 = ScoreBreakdown01(
+            pass_rate=breakdown.get("passRate"),
+            safety=breakdown.get("safety"),
+            judge=breakdown.get("judge"),
+            schema=breakdown.get("schema"),
+            latency=breakdown.get("latency"),
+            cost=breakdown.get("cost"),
+        )
+
+    contrib_pts = None
+    if explain and breakdown_01:
+        contrib_pts = compute_contrib_pts(breakdown_01)
+
+    thresholds = GateThresholds(
+        min_score=min_score,
+        max_drop=max_drop,
+        warn_drop=warn_drop,
+        min_n=min_n,
+        allow_weak_evidence=allow_weak_evidence,
+        baseline=baseline,
+        max_cost_usd=max_cost_usd,
+        max_latency_ms=max_latency_ms,
+        max_cost_delta_usd=max_cost_delta_usd,
+    )
+
+    policy_evidence = None
+    if explain and gate_result.get("policy_evidence", gate_result.get("policyEvidence")):
+        pe = gate_result.get("policy_evidence") or gate_result.get("policyEvidence")
+        policy_evidence = {
+            "failedCheck": pe.get("failed_check") or pe.get("failedCheck"),
+            "remediation": pe.get("remediation"),
+            "snapshot": pe.get("snapshot"),
+        }
+
+    return CheckReport(
+        schema_version=CHECK_REPORT_SCHEMA_VERSION,
+        evaluation_id=evaluation_id,
+        run_id=evaluation_run_id,
+        verdict=verdict,
+        gate_applied=gate_applied,
+        gate_mode=gate_mode,
+        actionable_message=actionable_message,
+        share_url=share_url,
+        policy=policy,
+        baseline_run_id=baseline_run_id or quality.get("baselineRunId") or quality.get("baseline_run_id"),
+        ci_run_url=ci_run_url,
+        reason_code=reason_code,
+        reason_message=gate_result.get("reason_message") or gate_result.get("reasonMessage"),
+        score=score,
+        baseline_score=baseline_score,
+        delta=regression_delta,
+        n=total,
+        evidence_level=quality.get("evidenceLevel") or quality.get("evidence_level"),
+        baseline_missing=quality.get("baselineMissing") or quality.get("baseline_missing"),
+        baseline_status=(
+            "missing"
+            if quality.get("baselineMissing") or quality.get("baseline_missing")
+            else ("found" if baseline_score is not None else None)
+        ),
+        flags=flags if flags else None,
+        breakdown_01=breakdown_01,
+        contrib_pts=contrib_pts,
+        thresholds=thresholds,
+        dashboard_url=dashboard_url,
+        failed_cases=failed_cases,
+        failed_cases_shown=failed_cases_shown,
+        failed_cases_more=failed_cases_more if failed_cases_more and failed_cases_more > 0 else None,
+        request_id=request_id,
+        explain=explain if explain else None,
+        policy_evidence=policy_evidence,
+    )