evalgate-ai 0.1.0__py3-none-any.whl

evalgate/__init__.py

@@ -0,0 +1,8 @@
+"""EvalGate: AI regression testing and release decision engine."""
+
+from evalgate._version import __version__
+from evalgate.api import compare
+from evalgate.io import EvalRun, InputError
+from evalgate.models import Report
+
+__all__ = ["EvalRun", "InputError", "Report", "__version__", "compare"]

evalgate/_version.py

@@ -0,0 +1,2 @@
+# Single source of the version: pyproject.toml reads it via [tool.hatch.version].
+__version__ = "0.1.0"

evalgate/api.py

@@ -0,0 +1,31 @@
+"""Path-based entry point shared by the CLI and the SDK, so both produce identical reports."""
+
+from pathlib import Path
+
+from evalgate.engine import evaluate, invalid_report
+from evalgate.io import InputError, load_policy, load_run
+from evalgate.models import Diagnostic, Policy, Report, Run
+
+
+def compare(
+    baseline: str | Path,
+    candidate: str | Path,
+    policy: str | Path = "evalgate.yaml",
+) -> Report:
+    """Load two run directories and a policy, then decide. Bad input yields an INVALID report."""
+    diagnostics: list[Diagnostic] = []
+    runs: list[Run] = []
+    loaded_policy: Policy | None = None
+    for run_dir in (baseline, candidate):
+        try:
+            runs.append(load_run(Path(run_dir)))
+        except InputError as exc:
+            diagnostics += exc.diagnostics
+    try:
+        loaded_policy = load_policy(Path(policy))
+    except InputError as exc:
+        diagnostics += exc.diagnostics
+
+    if diagnostics or loaded_policy is None:
+        return invalid_report(diagnostics)
+    return evaluate(runs[0], runs[1], loaded_policy)

evalgate/cli.py

@@ -0,0 +1,116 @@
+"""EvalGate command-line interface.
+
+Exit codes: 0 = PASS, 1 = FAIL (policy violated), 2 = INVALID input or tool error.
+"""
+
+import functools
+from collections.abc import Callable
+from pathlib import Path
+from typing import Annotated, ParamSpec
+
+import typer
+from rich.console import Console
+
+from evalgate._version import __version__
+from evalgate.api import compare as run_compare
+from evalgate.io import (
+    InputError,
+    load_run,
+    write_html_report,
+    write_json_report,
+    write_policy_template,
+)
+from evalgate.reporting.terminal import render, render_diagnostics
+from evalgate.validation import validate_run
+
+EXIT_PASS, EXIT_FAIL, EXIT_INVALID = 0, 1, 2
+EXIT_CODES = {"PASS": EXIT_PASS, "FAIL": EXIT_FAIL, "INVALID": EXIT_INVALID}
+
+app = typer.Typer(no_args_is_help=True, add_completion=False)
+console = Console()
+
+P = ParamSpec("P")
+
+
+def _fail_closed(command: Callable[P, None]) -> Callable[P, None]:
+    """Unexpected errors exit 2, never 1, so a crash can't be mistaken for a policy FAIL."""
+
+    @functools.wraps(command)
+    def wrapper(*args: P.args, **kwargs: P.kwargs) -> None:
+        try:
+            command(*args, **kwargs)
+        except (typer.Exit, typer.Abort):
+            raise
+        except Exception as exc:
+            typer.echo(f"internal error: {exc!r}", err=True)
+            raise typer.Exit(EXIT_INVALID) from exc
+
+    return wrapper
+
+
+def _version(value: bool) -> None:
+    if value:
+        typer.echo(f"evalgate {__version__}")
+        raise typer.Exit()
+
+
+@app.callback()
+def main(
+    version: Annotated[bool, typer.Option("--version", callback=_version, is_eager=True)] = False,
+) -> None:
+    """Compare evaluation runs and decide whether a candidate is releasable."""
+
+
+@app.command()
+@_fail_closed
+def validate(run_dir: Path) -> None:
+    """Check a run directory's structure, completeness and metric validity."""
+    try:
+        run = load_run(run_dir)
+        diagnostics = validate_run(run, str(run_dir))
+    except InputError as exc:
+        diagnostics = exc.diagnostics
+    render_diagnostics(diagnostics, console)
+    if any(d.level == "error" for d in diagnostics):
+        console.print("[bold yellow]INVALID[/]")
+        raise typer.Exit(EXIT_INVALID)
+    console.print(f"[bold green]valid[/]: {run.manifest.run_id}, {len(run.cases)} cases")
+
+
+@app.command()
+@_fail_closed
+def compare(
+    baseline: Path,
+    candidate: Path,
+    policy: Annotated[Path, typer.Option("--policy")] = Path("evalgate.yaml"),
+    json_out: Annotated[
+        Path | None, typer.Option("--json", help="Write the machine-readable report here.")
+    ] = None,
+    html_out: Annotated[
+        Path | None, typer.Option("--report", help="Write a standalone HTML report here.")
+    ] = None,
+) -> None:
+    """Compare a candidate run against a baseline under a release policy."""
+    report = run_compare(baseline, candidate, policy)
+    # Artifacts first: a terminal rendering problem must not lose the machine-readable result.
+    if json_out:
+        write_json_report(report, json_out)
+    if html_out:
+        write_html_report(report, html_out)
+    render(report, console)
+    raise typer.Exit(EXIT_CODES[report.status])
+
+
+@app.command()
+@_fail_closed
+def init(
+    path: Annotated[Path, typer.Argument()] = Path("evalgate.yaml"),
+    force: Annotated[bool, typer.Option("--force", help="Overwrite an existing file.")] = False,
+) -> None:
+    """Write a documented example release policy."""
+    try:
+        write_policy_template(path, force)
+    except FileExistsError as exc:
+        typer.echo(str(exc), err=True)
+        raise typer.Exit(EXIT_INVALID) from exc
+    console.print(f"wrote {path}: edit metrics and checks to match your cases.jsonl")

evalgate/comparison/__init__.py

@@ -0,0 +1,5 @@
+"""Pure baseline-vs-candidate comparison: case matching and metric deltas."""
+
+from evalgate.comparison.engine import compare
+
+__all__ = ["compare"]

evalgate/comparison/engine.py

@@ -0,0 +1,108 @@
+"""Paired baseline-vs-candidate comparison. Assumes inputs passed `validate_pair`."""
+
+from collections import defaultdict
+from statistics import fmean
+
+from evalgate.models import (
+    CaseDelta,
+    CaseResult,
+    CategoryDelta,
+    CheckChange,
+    Comparison,
+    MetricDelta,
+    MetricRule,
+    Run,
+)
+
+
+def _worse(delta: float, rule: MetricRule) -> bool:
+    return delta < 0 if rule.direction == "maximize" else delta > 0
+
+
+def _value(case: CaseResult, rule: MetricRule) -> float:
+    value = case.value(rule.path)
+    if value is None:  # validate_pair guarantees presence
+        raise ValueError(f"{case.case_id}: missing {rule.path}")
+    return value
+
+
+def compare(baseline: Run, candidate: Run, metrics: dict[str, MetricRule]) -> Comparison:
+    """Compare matched cases. Means and deltas use only cases `ok` in both runs."""
+    base = {c.case_id: c for c in baseline.cases}
+    cand = {c.case_id: c for c in candidate.cases}
+    matched = sorted(base.keys() & cand.keys())
+    pairs = [(base[i], cand[i]) for i in matched if base[i].status == cand[i].status == "ok"]
+
+    metric_deltas: list[MetricDelta] = []
+    regressed: list[CaseDelta] = []
+    improved: list[CaseDelta] = []
+    by_category: dict[tuple[str, str], list[tuple[float, float]]] = defaultdict(list)
+    for name, rule in sorted(metrics.items()):
+        values = [(_value(b, rule), _value(c, rule)) for b, c in pairs]
+        base_mean = fmean(b for b, _ in values)
+        cand_mean = fmean(c for _, c in values)
+        delta = cand_mean - base_mean
+        metric_deltas.append(
+            MetricDelta(
+                name=name,
+                path=rule.path,
+                direction=rule.direction,
+                cases=len(values),
+                baseline_mean=base_mean,
+                candidate_mean=cand_mean,
+                delta_abs=delta,
+                delta_pct=delta / abs(base_mean) * 100 if base_mean != 0 else None,
+            )
+        )
+        for (b_case, _), (b, c) in zip(pairs, values, strict=True):
+            if c != b:
+                change = CaseDelta(
+                    case_id=b_case.case_id, metric=name, baseline=b, candidate=c, delta_abs=c - b
+                )
+                (regressed if _worse(c - b, rule) else improved).append(change)
+            category = b_case.metadata.get("category")
+            if isinstance(category, str):
+                by_category[(category, name)].append((b, c))
+
+    # Largest change first within each metric; case_id breaks ties deterministically.
+    def order(d: CaseDelta) -> tuple[str, float, str]:
+        return (d.metric, -abs(d.delta_abs), d.case_id)
+
+    newly_failing: list[CheckChange] = []
+    newly_passing: list[CheckChange] = []
+    for b_case, c_case in pairs:
+        for check in sorted(b_case.checks.keys() & c_case.checks.keys()):
+            was, now = b_case.checks[check], c_case.checks[check]
+            if was != now:
+                flip = CheckChange(case_id=b_case.case_id, check=check)
+                (newly_failing if was else newly_passing).append(flip)
+
+    return Comparison(
+        matched_cases=len(matched),
+        compared_cases=len(pairs),
+        baseline_errors=[i for i in matched if base[i].status == "error"],
+        candidate_errors=[i for i in sorted(cand) if cand[i].status == "error"],
+        metrics=metric_deltas,
+        regressed_cases=sorted(regressed, key=order),
+        improved_cases=sorted(improved, key=order),
+        newly_failing_checks=newly_failing,
+        newly_passing_checks=newly_passing,
+        candidate_failing_checks=[
+            CheckChange(case_id=i, check=check)
+            for i in sorted(cand)
+            if cand[i].status == "ok"
+            for check, passed in sorted(cand[i].checks.items())
+            if not passed
+        ],
+        categories=[
+            CategoryDelta(
+                category=category,
+                metric=metric,
+                cases=len(values),
+                baseline_mean=fmean(b for b, _ in values),
+                candidate_mean=fmean(c for _, c in values),
+                delta_abs=fmean(c for _, c in values) - fmean(b for b, _ in values),
+            )
+            for (category, metric), values in sorted(by_category.items())
+        ],
+    )

evalgate/engine.py

@@ -0,0 +1,67 @@
+"""The one decision pipeline shared by CLI and SDK: validate -> compare -> apply policy."""
+
+import math
+
+from evalgate._version import __version__
+from evalgate.comparison import compare
+from evalgate.models import Comparison, Diagnostic, Policy, Report, Run
+from evalgate.policies import evaluate_policy
+from evalgate.validation import validate_pair, validate_run
+
+
+def _has_errors(diagnostics: list[Diagnostic]) -> bool:
+    return any(d.level == "error" for d in diagnostics)
+
+
+def _all_finite(c: Comparison) -> bool:
+    values = [x for m in c.metrics for x in (m.baseline_mean, m.candidate_mean, m.delta_abs)]
+    values += [m.delta_pct for m in c.metrics if m.delta_pct is not None]
+    values += [d.delta_abs for d in (*c.regressed_cases, *c.improved_cases)]
+    values += [x for g in c.categories for x in (g.baseline_mean, g.candidate_mean, g.delta_abs)]
+    return all(math.isfinite(x) for x in values)
+
+
+def invalid_report(diagnostics: list[Diagnostic], **identity: object) -> Report:
+    return Report.model_validate(
+        {"evalgate_version": __version__, "status": "INVALID", "diagnostics": diagnostics}
+        | identity
+    )
+
+
+def evaluate(baseline: Run, candidate: Run, policy: Policy) -> Report:
+    """Decide PASS / FAIL / INVALID. Never PASS on input that failed validation."""
+    identity = {"baseline": baseline.manifest, "candidate": candidate.manifest, "policy": policy}
+    diagnostics = validate_run(baseline, "baseline") + validate_run(candidate, "candidate")
+    if not _has_errors(diagnostics):
+        diagnostics += validate_pair(baseline, candidate, policy)
+    if _has_errors(diagnostics):
+        return invalid_report(diagnostics, **identity)
+
+    try:
+        comparison = compare(baseline, candidate, policy.metrics)
+    except OverflowError:
+        comparison = None
+    if comparison is None or not _all_finite(comparison):
+        overflow = Diagnostic(
+            level="error",
+            code="numeric_overflow",
+            message="metric values too large to average or subtract (result is not finite)",
+        )
+        return invalid_report([*diagnostics, overflow], **identity)
+
+    rules, rule_diagnostics = evaluate_policy(comparison, policy)
+    diagnostics += rule_diagnostics
+    if _has_errors(diagnostics):
+        status = "INVALID"
+    else:
+        status = "PASS" if all(r.passed for r in rules) else "FAIL"
+    return Report.model_validate(
+        {
+            "evalgate_version": __version__,
+            "status": status,
+            "diagnostics": diagnostics,
+            "comparison": comparison,
+            "rules": rules,
+        }
+        | identity
+    )

evalgate/io/__init__.py

@@ -0,0 +1,19 @@
+"""Reading and writing run artifacts (run.json + cases.jsonl). Only place that touches files."""
+
+from evalgate.io.reader import InputError, load_policy, load_run
+from evalgate.io.writer import (
+    EvalRun,
+    write_html_report,
+    write_json_report,
+    write_policy_template,
+)
+
+__all__ = [
+    "EvalRun",
+    "InputError",
+    "load_policy",
+    "load_run",
+    "write_html_report",
+    "write_json_report",
+    "write_policy_template",
+]

evalgate/io/reader.py

@@ -0,0 +1,110 @@
+"""Load run directories and policy files. Problems become diagnostics, not tracebacks."""
+
+from pathlib import Path
+from typing import Any, TypeVar
+
+import yaml
+from pydantic import BaseModel, ValidationError
+
+from evalgate.models import CaseResult, Diagnostic, Policy, Run, RunManifest
+
+
+class InputError(Exception):
+    """Input could not be loaded. Carries every diagnostic found, not just the first."""
+
+    def __init__(self, diagnostics: list[Diagnostic]) -> None:
+        super().__init__("; ".join(d.message for d in diagnostics))
+        self.diagnostics = diagnostics
+
+
+def _error(code: str, message: str) -> Diagnostic:
+    return Diagnostic(level="error", code=code, message=message)
+
+
+def _schema_errors(source: str, exc: ValidationError) -> list[Diagnostic]:
+    return [
+        _error("schema", f"{source}: {'.'.join(map(str, e['loc'])) or '<root>'}: {e['msg']}")
+        for e in exc.errors()
+    ]
+
+
+M = TypeVar("M", bound=BaseModel)
+
+
+def _parse(model: type[M], data: str | Any, source: str) -> M:
+    try:
+        if isinstance(data, str):
+            return model.model_validate_json(data)
+        return model.model_validate(data)
+    except ValidationError as exc:
+        raise InputError(_schema_errors(source, exc)) from exc
+
+
+def _read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8")
+    except FileNotFoundError as exc:
+        raise InputError([_error("file_missing", f"{path}: file not found")]) from exc
+    except (OSError, UnicodeDecodeError) as exc:
+        raise InputError([_error("file_unreadable", f"{path}: {exc}")]) from exc
+
+
+def load_run(run_dir: Path) -> Run:
+    """Read `run.json` + `cases.jsonl`. Raises InputError listing every bad file and line."""
+    if not run_dir.is_dir():
+        raise InputError([_error("run_dir_missing", f"{run_dir}: not a directory")])
+
+    diagnostics: list[Diagnostic] = []
+    manifest: RunManifest | None = None
+    cases: list[CaseResult] = []
+
+    manifest_path = run_dir / "run.json"
+    try:
+        manifest = _parse(RunManifest, _read_text(manifest_path), str(manifest_path))
+    except InputError as exc:
+        diagnostics += exc.diagnostics
+
+    cases_path = run_dir / "cases.jsonl"
+    try:
+        lines = _read_text(cases_path).splitlines()
+    except InputError as exc:
+        diagnostics += exc.diagnostics
+    else:
+        bad_lines = 0
+        for lineno, line in enumerate(lines, start=1):
+            if not line.strip():
+                continue
+            try:
+                cases.append(_parse(CaseResult, line, f"{cases_path}:{lineno}"))
+            except InputError as exc:
+                diagnostics += exc.diagnostics
+                bad_lines += 1
+        if not cases and not bad_lines:
+            diagnostics.append(_error("no_cases", f"{cases_path}: contains no cases"))
+
+    if diagnostics or manifest is None:
+        raise InputError(diagnostics)
+    return Run(manifest=manifest, cases=cases)
+
+
+class _UniqueKeyLoader(yaml.SafeLoader):
+    """SafeLoader that rejects duplicate keys; PyYAML keeps the last one, silently."""
+
+    def construct_mapping(self, node: yaml.MappingNode, deep: bool = False) -> dict[Any, Any]:
+        seen = set()
+        for key_node, _ in node.value:
+            key = self.construct_object(key_node, deep=deep)
+            if key in seen:
+                raise yaml.constructor.ConstructorError(
+                    None, None, f"duplicate key {key!r}", key_node.start_mark
+                )
+            seen.add(key)
+        return super().construct_mapping(node, deep=deep)
+
+
+def load_policy(path: Path) -> Policy:
+    try:
+        data = yaml.load(_read_text(path), Loader=_UniqueKeyLoader)  # SafeLoader subclass
+    except yaml.YAMLError as exc:
+        raise InputError([_error("policy_yaml", f"{path}: invalid YAML: {exc}")]) from exc
+    return _parse(Policy, data, str(path))

evalgate/io/writer.py

@@ -0,0 +1,139 @@
+import os
+from pathlib import Path
+from types import TracebackType
+from typing import IO, Any, Self
+
+from evalgate.models import SCHEMA_VERSION, CaseResult, Report, RunManifest
+from evalgate.models.policy import POLICY_TEMPLATE
+
+
+def _atomic_write(path: Path, text: str) -> None:
+    tmp = path.with_name(f".{path.name}.tmp")
+    tmp.write_text(text, encoding="utf-8")
+    os.replace(tmp, path)
+
+
+def write_json_report(report: Report, path: Path) -> None:
+    _atomic_write(path, report.model_dump_json(indent=2) + "\n")
+
+
+def write_html_report(report: Report, path: Path) -> None:
+    from evalgate.reporting.html import render_html
+
+    _atomic_write(path, render_html(report))
+
+
+def write_policy_template(path: Path, force: bool = False) -> None:
+    if path.exists() and not force:
+        raise FileExistsError(f"{path} already exists; pass --force to overwrite")
+    _atomic_write(path, POLICY_TEMPLATE)
+
+
+class EvalRun:
+    """Record case results into a run directory (`run.json` + `cases.jsonl`).
+
+    Cases stream to a partial file; only a clean exit from the `with` block publishes
+    them and writes `run.json` last. An interrupted run therefore never loads as complete,
+    and with `overwrite=True` a crash leaves the previous run untouched.
+    """
+
+    def __init__(
+        self,
+        run_id: str,
+        *,
+        dataset: str,
+        dataset_version: str,
+        evaluator_version: str,
+        path: str | Path | None = None,
+        application: str | None = None,
+        application_version: str | None = None,
+        overwrite: bool = False,
+        **metadata: Any,
+    ) -> None:
+        clash = sorted(metadata.keys() & RunManifest.model_fields.keys())
+        if clash:  # e.g. dataset_id= would silently override dataset=
+            raise ValueError(f"metadata keys collide with manifest fields: {', '.join(clash)}")
+        self.manifest = RunManifest.model_validate(
+            {
+                "schema_version": SCHEMA_VERSION,
+                "run_id": run_id,
+                "dataset_id": dataset,
+                "dataset_version": dataset_version,
+                "evaluator_version": evaluator_version,
+                "application": application,
+                "application_version": application_version,
+            }
+            | metadata
+        )
+        self.path = Path(path) if path is not None else Path("evals") / run_id
+        self.overwrite = overwrite
+        self._partial = self.path / ".cases.jsonl.partial"
+        self._file: IO[str] | None = None
+        self._ids: set[str] = set()
+
+    def __enter__(self) -> Self:
+        exists = (self.path / "run.json").exists() or (self.path / "cases.jsonl").exists()
+        if exists and not self.overwrite:
+            raise FileExistsError(f"{self.path} already holds a run; pass overwrite=True")
+        self.path.mkdir(parents=True, exist_ok=True)
+        self._ids = set()
+        self._file = self._partial.open("w", encoding="utf-8")
+        return self
+
+    def __exit__(
+        self,
+        exc_type: type[BaseException] | None,
+        exc: BaseException | None,
+        tb: TracebackType | None,
+    ) -> None:
+        assert self._file is not None
+        self._file.close()
+        self._file = None
+        if exc_type is not None:
+            return  # the partial file is never loaded; any previous run stays intact
+        # Old run.json goes first so old manifest + new cases can never load together.
+        (self.path / "run.json").unlink(missing_ok=True)
+        os.replace(self._partial, self.path / "cases.jsonl")
+        _atomic_write(self.path / "run.json", self.manifest.model_dump_json(indent=2) + "\n")
+
+    def _write(self, case: CaseResult) -> None:
+        if self._file is None:
+            raise RuntimeError("EvalRun.record() must be called inside `with EvalRun(...)`")
+        if case.case_id in self._ids:
+            raise ValueError(f"duplicate case_id: {case.case_id}")
+        self._ids.add(case.case_id)
+        self._file.write(case.model_dump_json(exclude_defaults=True) + "\n")
+        self._file.flush()
+
+    def record(
+        self,
+        case_id: str,
+        *,
+        scores: dict[str, float] | None = None,
+        checks: dict[str, bool] | None = None,
+        metrics: dict[str, float] | None = None,
+        metadata: dict[str, Any] | None = None,
+    ) -> None:
+        """Record a successful case. Values are validated now, not at compare time."""
+        self._write(
+            CaseResult.model_validate(
+                {
+                    "case_id": case_id,
+                    "status": "ok",
+                    "scores": scores or {},
+                    "checks": checks or {},
+                    "metrics": metrics or {},
+                    "metadata": metadata or {},
+                }
+            )
+        )
+
+    def record_error(
+        self, case_id: str, error: str, *, metadata: dict[str, Any] | None = None
+    ) -> None:
+        """Record that the pipeline failed on a case; never invent placeholder scores."""
+        self._write(
+            CaseResult.model_validate(
+                {"case_id": case_id, "status": "error", "error": error, "metadata": metadata or {}}
+            )
+        )

evalgate/models/__init__.py

@@ -0,0 +1,35 @@
+"""Pydantic models: run manifest, case result, release policy, report."""
+
+from evalgate.models.policy import CheckRules, ComparisonRules, MetricRule, Policy
+from evalgate.models.report import (
+    CaseDelta,
+    CategoryDelta,
+    CheckChange,
+    Comparison,
+    Diagnostic,
+    MetricDelta,
+    Report,
+    RuleResult,
+    Status,
+)
+from evalgate.models.run import SCHEMA_VERSION, CaseResult, Run, RunManifest
+
+__all__ = [
+    "SCHEMA_VERSION",
+    "CaseDelta",
+    "CaseResult",
+    "CategoryDelta",
+    "CheckChange",
+    "CheckRules",
+    "Comparison",
+    "ComparisonRules",
+    "Diagnostic",
+    "MetricDelta",
+    "MetricRule",
+    "Policy",
+    "Report",
+    "RuleResult",
+    "Run",
+    "RunManifest",
+    "Status",
+]