evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/runtime/run_report.py

@@ -0,0 +1,249 @@
+"""Deterministic RunReport serialization (T4).
+
+Port of the TypeScript SDK's ``run-report.ts``.
+Provides a stable report format for downstream processing (explain, diff, history).
+"""
+
+from __future__ import annotations
+
+import json
+import platform
+import sys
+from dataclasses import asdict, dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+
+RUN_REPORT_SCHEMA_VERSION = "1"
+
+
+@dataclass
+class RunResult:
+    """Individual test result."""
+
+    test_id: str
+    test_name: str
+    file_path: str
+    position: dict[str, int]
+    input: str
+    passed: bool
+    score: float
+    duration_ms: float
+    metadata: dict[str, Any] | None = None
+    tags: list[str] = field(default_factory=list)
+    assertions: list[dict[str, Any]] = field(default_factory=list)
+
+
+@dataclass
+class RunFailure:
+    """Failure or error information."""
+
+    test_id: str
+    test_name: str
+    file_path: str
+    position: dict[str, int]
+    classification: str  # "failed" | "error" | "timeout"
+    message: str
+    timestamp: str = ""
+    error_envelope: dict[str, Any] | None = None
+
+
+@dataclass
+class RunSummary:
+    """Execution summary statistics."""
+
+    total: int = 0
+    passed: int = 0
+    failed: int = 0
+    errors: int = 0
+    timeouts: int = 0
+    pass_rate: float = 0.0
+    average_score: float = 0.0
+    total_duration_ms: float = 0.0
+    success: bool = True
+
+
+@dataclass
+class RunConfig:
+    """Execution configuration."""
+
+    executor_type: str = "local"
+    max_parallel: int | None = None
+    default_timeout: int = 30_000
+    environment: dict[str, str] = field(default_factory=dict)
+
+
+@dataclass
+class RunReport:
+    """Main run report structure."""
+
+    schema_version: str = RUN_REPORT_SCHEMA_VERSION
+    run_id: str = ""
+    started_at: str = ""
+    finished_at: str = ""
+    runtime: dict[str, str] = field(default_factory=dict)
+    results: list[RunResult] = field(default_factory=list)
+    failures: list[RunFailure] = field(default_factory=list)
+    summary: RunSummary = field(default_factory=RunSummary)
+    config: RunConfig = field(default_factory=RunConfig)
+
+    def to_json(self) -> str:
+        """Serialize to deterministic JSON string."""
+        return json.dumps(asdict(self), indent=2, default=str)
+
+
+class RunReportBuilder:
+    """Builder for creating deterministic RunReport instances."""
+
+    def __init__(
+        self,
+        run_id: str,
+        runtime_info: dict[str, str],
+    ) -> None:
+        self._report = RunReport(
+            run_id=run_id,
+            started_at=datetime.now(timezone.utc).isoformat(),
+            runtime=runtime_info,
+            config=RunConfig(
+                environment={
+                    "python_version": sys.version.split()[0],
+                    "platform": platform.system().lower(),
+                    "arch": platform.machine(),
+                },
+            ),
+        )
+        self._scores: list[float] = []
+
+    def add_result(
+        self,
+        test_id: str,
+        test_name: str,
+        file_path: str,
+        position: dict[str, int],
+        input: str,
+        *,
+        passed: bool,
+        score: float,
+        duration_ms: float = 0.0,
+        metadata: dict[str, Any] | None = None,
+        tags: list[str] | None = None,
+        assertions: list[dict[str, Any]] | None = None,
+        classification: str = "passed",
+        error: str | None = None,
+        error_envelope: dict[str, Any] | None = None,
+    ) -> None:
+        """Add a test result to the report."""
+        result = RunResult(
+            test_id=test_id,
+            test_name=test_name,
+            file_path=file_path,
+            position=position,
+            input=input,
+            passed=passed,
+            score=score,
+            duration_ms=duration_ms,
+            metadata=metadata,
+            tags=tags or [],
+            assertions=assertions or [],
+        )
+        self._report.results.append(result)
+
+        # Update summary
+        s = self._report.summary
+        s.total += 1
+        s.total_duration_ms += duration_ms
+
+        if passed:
+            s.passed += 1
+        elif classification == "error":
+            s.errors += 1
+            s.success = False
+        elif classification == "timeout":
+            s.timeouts += 1
+            s.success = False
+        else:
+            s.failed += 1
+            s.success = False
+
+        s.pass_rate = (s.passed / s.total * 100) if s.total > 0 else 0.0
+        if score > 0:
+            self._scores.append(score)
+        s.average_score = (sum(self._scores) / len(self._scores)) if self._scores else 0.0
+
+        # Add to failures if needed
+        if not passed or classification in ("error", "timeout"):
+            failure = RunFailure(
+                test_id=test_id,
+                test_name=test_name,
+                file_path=file_path,
+                position=position,
+                classification=classification if classification in ("error", "timeout") else "failed",
+                message=error or "Test failed",
+                timestamp=datetime.now(timezone.utc).isoformat(),
+                error_envelope=error_envelope,
+            )
+            self._report.failures.append(failure)
+
+    def set_config(self, **kwargs: Any) -> None:
+        """Update execution configuration fields."""
+        for key, value in kwargs.items():
+            if hasattr(self._report.config, key):
+                setattr(self._report.config, key, value)
+
+    def build(self) -> RunReport:
+        """Finalize and return the complete report."""
+        # Sort for determinism
+        self._report.results.sort(key=lambda r: r.test_id)
+        self._report.failures.sort(key=lambda f: f.test_id)
+        self._report.finished_at = datetime.now(timezone.utc).isoformat()
+        return self._report
+
+    def to_json(self) -> str:
+        """Build and serialize to JSON."""
+        return self.build().to_json()
+
+    async def write_to_file(self, file_path: str) -> None:
+        """Write report to file."""
+        from pathlib import Path
+
+        Path(file_path).write_text(self.to_json(), encoding="utf-8")
+
+
+def create_run_report(
+    run_id: str,
+    runtime_info: dict[str, str],
+) -> RunReportBuilder:
+    """Create a new RunReport builder."""
+    return RunReportBuilder(run_id, runtime_info)
+
+
+def _filter_dataclass_fields(cls: type, d: dict[str, Any]) -> dict[str, Any]:
+    """Keep only keys that match dataclass field names — prevents TypeError on extras."""
+    import dataclasses
+
+    valid = {f.name for f in dataclasses.fields(cls)}
+    return {k: v for k, v in d.items() if k in valid}
+
+
+def parse_run_report(json_str: str) -> RunReport:
+    """Parse a RunReport from a JSON string."""
+    data = json.loads(json_str)
+    version = data.get("schema_version", "")
+    if version != RUN_REPORT_SCHEMA_VERSION:
+        raise ValueError(f"Unsupported RunReport schema version: {version}")
+
+    summary = RunSummary(**_filter_dataclass_fields(RunSummary, data.get("summary", {})))
+    config = RunConfig(**_filter_dataclass_fields(RunConfig, data.get("config", {})))
+    results = [RunResult(**_filter_dataclass_fields(RunResult, r)) for r in data.get("results", [])]
+    failures = [RunFailure(**_filter_dataclass_fields(RunFailure, f)) for f in data.get("failures", [])]
+
+    return RunReport(
+        schema_version=data["schema_version"],
+        run_id=data["run_id"],
+        started_at=data.get("started_at", ""),
+        finished_at=data.get("finished_at", ""),
+        runtime=data.get("runtime", {}),
+        results=results,
+        failures=failures,
+        summary=summary,
+        config=config,
+    )

evalgate_sdk/runtime/types.py

@@ -0,0 +1,143 @@
+"""Core types for the runtime foundation."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from typing import Any, Literal
+
+
+@dataclass
+class DependsOn:
+    """Dependency hints for impact analysis."""
+
+    prompts: list[str] = field(default_factory=list)
+    datasets: list[str] = field(default_factory=list)
+    tools: list[str] = field(default_factory=list)
+    code: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SpecOptions:
+    timeout_ms: int = 30_000
+    retries: int = 0
+    tags: list[str] = field(default_factory=list)
+    skip: bool = False
+    only: bool = False
+    description: str | None = None
+    budget: str | None = None
+    model: str | None = None
+    metadata: dict[str, Any] | None = None
+    depends_on: DependsOn | None = None
+
+
+@dataclass
+class SpecConfig:
+    name: str
+    executor: Any = None
+    options: SpecOptions = field(default_factory=SpecOptions)
+    description: str | None = None
+    suite: str | None = None
+    tags: list[str] | None = None
+    timeout: int | None = None
+    retries: int | None = None
+    budget: str | None = None
+    model: str | None = None
+    metadata: dict[str, Any] | None = None
+    depends_on: DependsOn | None = None
+
+
+@dataclass
+class EvalSpec:
+    id: str
+    name: str
+    executor: Any
+    options: SpecOptions = field(default_factory=SpecOptions)
+    file_path: str | None = None
+    suite: str | None = None
+    description: str | None = None
+    position: dict[str, int] | None = None
+    tags: list[str] = field(default_factory=list)
+    metadata: dict[str, Any] | None = None
+    config: dict[str, Any] | None = None
+    mode: Literal["normal", "skip", "only"] = "normal"
+
+
+@dataclass
+class EvalContext:
+    input: Any = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+    options: dict[str, Any] = field(default_factory=dict)
+    trace_id: str | None = None
+
+
+@dataclass
+class EvalResult:
+    passed: bool
+    score: float = 0.0
+    assertions: list[Any] = field(default_factory=list)
+    metadata: dict[str, Any] = field(default_factory=dict)
+    error: str | None = None
+    duration_ms: float = 0.0
+    status: Literal["passed", "failed", "error", "timeout"] = "passed"
+    output: str | None = None
+    tokens: int | None = None
+
+
+@dataclass
+class ExecutorCapabilities:
+    supports_async: bool = True
+    supports_timeout: bool = True
+    supports_retries: bool = True
+    supports_parallel: bool = False
+
+
+@dataclass
+class ExecutionErrorEnvelope:
+    error_type: str
+    message: str
+    stack: str | None = None
+    retryable: bool = False
+
+
+@dataclass
+class RuntimeHealth:
+    status: Literal["healthy", "degraded", "unhealthy"] = "healthy"
+    spec_count: int = 0
+    memory_estimate_mb: float = 0.0
+    uptime_ms: float = 0.0
+
+
+# ── Error classes ────────────────────────────────────────────────────
+
+
+class EvalRuntimeError(Exception):
+    """Base error for runtime operations."""
+
+    pass
+
+
+class SpecRegistrationError(EvalRuntimeError):
+    """Raised when a spec fails to register."""
+
+    pass
+
+
+class SpecExecutionError(EvalRuntimeError):
+    """Raised when a spec fails to execute."""
+
+    pass
+
+
+class RuntimeError(EvalRuntimeError):
+    """Raised for general runtime errors."""
+
+    pass
+
+
+class EvalExecutionError(EvalRuntimeError):
+    """Raised during eval execution with context."""
+
+    def __init__(self, message: str, spec_id: str, cause: Exception | None = None) -> None:
+        super().__init__(message)
+        self.spec_id = spec_id
+        self.cause = cause

evalgate_sdk/snapshot.py

@@ -0,0 +1,219 @@
+"""Snapshot testing — save, load, compare LLM outputs against golden snapshots."""
+
+from __future__ import annotations
+
+import hashlib
+import json
+import re
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from pathlib import Path
+
+
+@dataclass
+class SnapshotMetadata:
+    name: str
+    created_at: str
+    content_hash: str
+    version: int = 1
+
+
+@dataclass
+class SnapshotData:
+    name: str
+    output: str
+    metadata: SnapshotMetadata
+    tags: list[str] = field(default_factory=list)
+
+
+@dataclass
+class SnapshotComparison:
+    name: str
+    matches: bool
+    similarity: float
+    current_output: str
+    snapshot_output: str
+    diff_lines: list[str] = field(default_factory=list)
+
+
+_DEFAULT_DIR = ".snapshots"
+
+
+def _safe_name(name: str) -> str:
+    """Sanitize snapshot name for filesystem use."""
+    safe = re.sub(r"[^\w\-.]", "_", name)
+    if ".." in safe or safe.startswith("/") or safe.startswith("\\"):
+        raise ValueError(f"Invalid snapshot name: {name}")
+    return safe
+
+
+def _content_hash(text: str) -> str:
+    return hashlib.sha256(text.encode()).hexdigest()
+
+
+def _similarity(a: str, b: str) -> float:
+    """Line-by-line similarity ratio."""
+    lines_a = a.splitlines()
+    lines_b = b.splitlines()
+    if not lines_a and not lines_b:
+        return 1.0
+    total = max(len(lines_a), len(lines_b))
+    matches = sum(1 for la, lb in zip(lines_a, lines_b, strict=False) if la == lb)
+    return matches / total if total > 0 else 1.0
+
+
+class SnapshotManager:
+    """Manage snapshot files on disk.
+
+    Usage::
+
+        mgr = SnapshotManager()
+        snap = mgr.save("my-test", "Hello world output")
+        comparison = mgr.compare("my-test", "Hello world output v2")
+    """
+
+    def __init__(self, directory: str = _DEFAULT_DIR) -> None:
+        self._dir = Path(directory)
+
+    def _path(self, name: str) -> Path:
+        return self._dir / f"{_safe_name(name)}.json"
+
+    def save(self, name: str, output: str, tags: list[str] | None = None) -> SnapshotData:
+        """Save a snapshot to disk."""
+        self._dir.mkdir(parents=True, exist_ok=True)
+        snap = SnapshotData(
+            name=name,
+            output=output,
+            metadata=SnapshotMetadata(
+                name=name,
+                created_at=datetime.now(timezone.utc).isoformat(),
+                content_hash=_content_hash(output),
+            ),
+            tags=tags or [],
+        )
+        path = self._path(name)
+        path.write_text(
+            json.dumps(
+                {
+                    "name": snap.name,
+                    "output": snap.output,
+                    "metadata": {
+                        "name": snap.metadata.name,
+                        "created_at": snap.metadata.created_at,
+                        "content_hash": snap.metadata.content_hash,
+                        "version": snap.metadata.version,
+                    },
+                    "tags": snap.tags,
+                },
+                indent=2,
+            ),
+            encoding="utf-8",
+        )
+        return snap
+
+    def load(self, name: str) -> SnapshotData | None:
+        """Load a snapshot from disk."""
+        path = self._path(name)
+        if not path.exists():
+            return None
+        raw = json.loads(path.read_text(encoding="utf-8"))
+        return SnapshotData(
+            name=raw["name"],
+            output=raw["output"],
+            metadata=SnapshotMetadata(**raw["metadata"]),
+            tags=raw.get("tags", []),
+        )
+
+    def compare(self, name: str, current_output: str) -> SnapshotComparison:
+        """Compare current output against a saved snapshot."""
+        existing = self.load(name)
+        if existing is None:
+            self.save(name, current_output)
+            return SnapshotComparison(
+                name=name,
+                matches=True,
+                similarity=1.0,
+                current_output=current_output,
+                snapshot_output=current_output,
+            )
+
+        matches = existing.output == current_output
+        sim = _similarity(existing.output, current_output)
+
+        diff: list[str] = []
+        old_lines = existing.output.splitlines()
+        new_lines = current_output.splitlines()
+        for i in range(max(len(old_lines), len(new_lines))):
+            old = old_lines[i] if i < len(old_lines) else ""
+            new = new_lines[i] if i < len(new_lines) else ""
+            if old != new:
+                diff.append(f"L{i + 1}: -{old!r} +{new!r}")
+
+        return SnapshotComparison(
+            name=name,
+            matches=matches,
+            similarity=sim,
+            current_output=current_output,
+            snapshot_output=existing.output,
+            diff_lines=diff,
+        )
+
+    def delete(self, name: str) -> bool:
+        path = self._path(name)
+        if path.exists():
+            path.unlink()
+            return True
+        return False
+
+    def list_snapshots(self) -> list[SnapshotData]:
+        if not self._dir.exists():
+            return []
+        results: list[SnapshotData] = []
+        for p in sorted(self._dir.glob("*.json")):
+            try:
+                raw = json.loads(p.read_text(encoding="utf-8"))
+                results.append(
+                    SnapshotData(
+                        name=raw["name"],
+                        output=raw["output"],
+                        metadata=SnapshotMetadata(**raw["metadata"]),
+                        tags=raw.get("tags", []),
+                    )
+                )
+            except (json.JSONDecodeError, KeyError):
+                continue
+        return results
+
+
+# Module-level convenience functions using a default manager
+
+_default_manager: SnapshotManager | None = None
+
+
+def _get_manager(directory: str | None = None) -> SnapshotManager:
+    global _default_manager
+    if directory is not None:
+        return SnapshotManager(directory)
+    if _default_manager is None:
+        _default_manager = SnapshotManager()
+    return _default_manager
+
+
+def snapshot(output: str, name: str, *, directory: str | None = None, tags: list[str] | None = None) -> SnapshotData:
+    return _get_manager(directory).save(name, output, tags)
+
+
+def load_snapshot(name: str, *, directory: str | None = None) -> SnapshotData | None:
+    return _get_manager(directory).load(name)
+
+
+def compare_with_snapshot(name: str, current_output: str, *, directory: str | None = None) -> SnapshotComparison:
+    return _get_manager(directory).compare(name, current_output)
+
+
+def delete_snapshot(name: str, *, directory: str | None = None) -> bool:
+    return _get_manager(directory).delete(name)
+
+
+def list_snapshots(*, directory: str | None = None) -> list[SnapshotData]:
+    return _get_manager(directory).list_snapshots()