evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/runtime/eval.py

@@ -0,0 +1,318 @@
+"""defineEval DSL — the primary API for declaring evaluation specs."""
+
+from __future__ import annotations
+
+import csv
+import hashlib
+import inspect
+import io
+import json
+import os
+import re
+from collections.abc import Callable
+from typing import Any, Literal
+
+from evalgate_sdk.runtime.registry import get_active_runtime
+from evalgate_sdk.runtime.types import (
+    EvalContext,
+    EvalResult,
+    EvalSpec,
+    SpecConfig,
+    SpecOptions,
+    SpecRegistrationError,
+)
+
+_NAME_PATTERN = re.compile(r"^[\w\s\-]{1,100}$")
+
+
+def _generate_spec_id(name: str, file_path: str | None = None) -> str:
+    """Generate a content-addressable spec ID."""
+    source = name
+    if file_path is None:
+        frame = inspect.stack()
+        for f in frame[1:]:
+            if "evalgate_sdk" not in f.filename:
+                file_path = f"{f.filename}:{f.lineno}"
+                break
+    if file_path:
+        source = f"{file_path}:{name}"
+    return hashlib.sha256(source.encode()).hexdigest()[:16]
+
+
+def _validate_name(name: str) -> None:
+    if not _NAME_PATTERN.match(name):
+        raise SpecRegistrationError(
+            f"Invalid spec name '{name}': must be 1-100 chars, alphanumeric/hyphens/underscores"
+        )
+
+
+def define_eval(
+    name_or_config: Any = None,
+    executor: Callable[..., Any] | None = None,
+    *,
+    name: str | None = None,
+    options: SpecOptions | None = None,
+    description: str | None = None,
+    suite: str | None = None,
+    tags: list[str] | None = None,
+    timeout_ms: int = 30_000,
+) -> EvalSpec | None:
+    """Register an eval spec with the active runtime.
+
+    Can be called as::
+
+        # Positional style
+        define_eval("my-test", my_executor)
+
+        # Config style
+        define_eval(SpecConfig(name="my-test", executor=my_executor))
+
+        # Decorator style
+        @define_eval(name="my-test")
+        async def my_test(ctx):
+            ...
+    """
+    if isinstance(name_or_config, SpecConfig):
+        cfg = name_or_config
+        spec_name = cfg.name
+        spec_executor = cfg.executor
+        spec_options = cfg.options
+        spec_desc = cfg.description
+        spec_suite = cfg.suite
+    elif isinstance(name_or_config, str):
+        spec_name = name_or_config
+        spec_executor = executor
+        spec_options = options or SpecOptions(timeout_ms=timeout_ms, tags=tags or [])
+        spec_desc = description
+        spec_suite = suite
+    elif name_or_config is None and name is not None:
+        spec_name = name
+        spec_executor = executor
+        spec_options = options or SpecOptions(timeout_ms=timeout_ms, tags=tags or [])
+        spec_desc = description
+        spec_suite = suite
+    elif callable(name_or_config) and name is not None:
+        spec_name = name
+        spec_executor = name_or_config
+        spec_options = options or SpecOptions(timeout_ms=timeout_ms, tags=tags or [])
+        spec_desc = description
+        spec_suite = suite
+    else:
+        # Decorator mode — return a decorator
+        def decorator(fn: Callable[..., Any]) -> Callable[..., Any]:
+            define_eval(
+                fn,
+                name=name or fn.__name__,
+                options=options,
+                description=description,
+                suite=suite,
+                tags=tags,
+                timeout_ms=timeout_ms,
+            )
+            return fn
+
+        if callable(name_or_config):
+            return decorator(name_or_config)
+        return decorator  # type: ignore[return-value]
+
+    _validate_name(spec_name)
+    spec_id = _generate_spec_id(spec_name)
+
+    spec = EvalSpec(
+        id=spec_id,
+        name=spec_name,
+        executor=spec_executor,
+        options=spec_options,
+        suite=spec_suite,
+        description=spec_desc,
+    )
+
+    runtime = get_active_runtime()
+    if runtime is not None:
+        runtime.register(spec)
+
+    return spec
+
+
+def _define_eval_with_mode(
+    mode: Literal["normal", "skip", "only"],
+    name_or_config: Any = None,
+    executor: Callable[..., Any] | None = None,
+    *,
+    name: str | None = None,
+    options: SpecOptions | None = None,
+    description: str | None = None,
+    suite: str | None = None,
+    tags: list[str] | None = None,
+    timeout_ms: int = 30_000,
+) -> EvalSpec | None:
+    """Internal: register with a specific mode."""
+    spec = define_eval(
+        name_or_config,
+        executor,
+        name=name,
+        options=options,
+        description=description,
+        suite=suite,
+        tags=tags,
+        timeout_ms=timeout_ms,
+    )
+    if isinstance(spec, EvalSpec):
+        spec.mode = mode
+    return spec
+
+
+def define_eval_skip(
+    name_or_config: Any = None,
+    executor: Callable[..., Any] | None = None,
+    **kwargs: Any,
+) -> EvalSpec | None:
+    """Register a spec but skip it during execution (vitest/jest ``.skip`` convention)."""
+    return _define_eval_with_mode("skip", name_or_config, executor, **kwargs)
+
+
+def define_eval_only(
+    name_or_config: Any = None,
+    executor: Callable[..., Any] | None = None,
+    **kwargs: Any,
+) -> EvalSpec | None:
+    """Register a spec for exclusive execution (vitest/jest ``.only`` convention)."""
+    return _define_eval_with_mode("only", name_or_config, executor, **kwargs)
+
+
+class _EvalAI:
+    """Convenience namespace — ``evalai.test`` is an alias for ``define_eval``."""
+
+    test = staticmethod(define_eval)
+
+
+evalai = _EvalAI()
+
+# Attach .skip / .only on define_eval to match TS defineEval.skip() / defineEval.only()
+define_eval.skip = define_eval_skip  # type: ignore[attr-defined]
+define_eval.only = define_eval_only  # type: ignore[attr-defined]
+
+
+# ── skip/only filtering ──────────────────────────────────────────────
+
+
+def get_filtered_specs(specs: list[EvalSpec]) -> list[EvalSpec]:
+    """Apply skip/only semantics to a list of specs.
+
+    If any spec has ``mode == "only"``, return only those.
+    Otherwise, return all specs except those with ``mode == "skip"``.
+    """
+    only_specs = [s for s in specs if s.mode == "only"]
+    if only_specs:
+        return only_specs
+    return [s for s in specs if s.mode != "skip"]
+
+
+# ── from_dataset ─────────────────────────────────────────────────────
+
+
+def _parse_jsonl(content: str) -> list[dict[str, Any]]:
+    rows: list[dict[str, Any]] = []
+    for i, line in enumerate(content.splitlines()):
+        line = line.strip()
+        if not line:
+            continue
+        try:
+            rows.append(json.loads(line))
+        except json.JSONDecodeError as exc:
+            raise SpecRegistrationError(f"Invalid JSON on line {i + 1} of dataset: {exc}") from exc
+    return rows
+
+
+def _parse_csv(content: str) -> list[dict[str, Any]]:
+    reader = csv.DictReader(io.StringIO(content))
+    return [dict(row) for row in reader]
+
+
+def from_dataset(
+    name: str,
+    dataset_path: str,
+    executor: Callable[..., Any],
+    options: SpecOptions | None = None,
+) -> None:
+    """Load a JSONL, CSV, or JSON dataset and register one spec per row.
+
+    Each row is passed as ``context.input`` to the executor.
+
+    Example::
+
+        from_dataset("rag-accuracy", "./evals/golden.jsonl", my_executor)
+    """
+    resolved = os.path.abspath(dataset_path)
+    if not os.path.isfile(resolved):
+        raise SpecRegistrationError(f"Dataset file not found: {resolved}")
+
+    with open(resolved, encoding="utf-8") as f:
+        content = f.read()
+
+    ext = os.path.splitext(resolved)[1].lower()
+    if ext in (".jsonl", ".ndjson"):
+        rows = _parse_jsonl(content)
+    elif ext == ".csv":
+        rows = _parse_csv(content)
+    elif ext == ".json":
+        parsed = json.loads(content)
+        rows = parsed if isinstance(parsed, list) else [parsed]
+    else:
+        raise SpecRegistrationError(f"Unsupported dataset format: {ext}. Use .jsonl, .ndjson, .csv, or .json")
+
+    if not rows:
+        raise SpecRegistrationError(f"Dataset is empty: {resolved}")
+
+    for i, row in enumerate(rows):
+        row_name = f"{name} - row {i + 1}"
+
+        def _make_wrapper(r: dict[str, Any]) -> Callable[..., Any]:
+            async def wrapper(ctx: EvalContext) -> EvalResult:
+                ctx.input = r
+                return await executor(ctx)
+
+            return wrapper
+
+        row_options = SpecOptions(
+            timeout_ms=options.timeout_ms if options else 30_000,
+            tags=list(options.tags) if options and options.tags else [],
+            metadata={
+                **(options.metadata or {} if options else {}),
+                "dataset_path": resolved,
+                "dataset_row": i + 1,
+            },
+        )
+        define_eval(row_name, _make_wrapper(row), options=row_options)
+
+
+def define_suite(name: str, specs: list[Callable[[], None]]) -> None:
+    """Group multiple define_eval calls into a named suite."""
+    for spec_fn in specs:
+        spec_fn()
+
+
+def create_result(
+    *,
+    passed: bool,
+    score: float = 0.0,
+    assertions: list[Any] | None = None,
+    metadata: dict[str, Any] | None = None,
+    error: str | None = None,
+    output: str | None = None,
+    tokens: int | None = None,
+    duration_ms: float | None = None,
+) -> EvalResult:
+    """Create an evaluation result."""
+    clamped_score = max(0.0, min(100.0, score))
+    return EvalResult(
+        passed=passed,
+        score=clamped_score,
+        assertions=assertions or [],
+        metadata=metadata or {},
+        error=error,
+        status="passed" if passed else ("error" if error else "failed"),
+        output=output,
+        tokens=tokens,
+        duration_ms=duration_ms or 0.0,
+    )

evalgate_sdk/runtime/execution_mode.py

@@ -0,0 +1,170 @@
+"""Dual-path execution mode detection (T8).
+
+Port of the TypeScript SDK's ``execution-mode.ts``.
+Environment flag ``EVALGATE_RUNTIME=legacy|spec|auto``.
+"""
+
+from __future__ import annotations
+
+import os
+from dataclasses import dataclass, field
+from pathlib import Path
+from typing import Literal
+
+ExecutionMode = Literal["legacy", "spec", "auto"]
+
+ENV_VAR = "EVALGATE_RUNTIME"
+POSSIBLE_VALUES = ("legacy", "spec", "auto")
+DEFAULT_MODE: ExecutionMode = "auto"
+
+
+@dataclass
+class ExecutionModeConfig:
+    mode: ExecutionMode = "auto"
+    has_spec_runtime: bool = False
+    has_legacy_runtime: bool = False
+    project_root: str = ""
+    spec_files: list[str] = field(default_factory=list)
+    legacy_config: str | None = None
+
+
+def find_spec_files(project_root: str) -> list[str]:
+    """Search for Python files containing ``define_eval`` calls."""
+    root = Path(project_root)
+    patterns = [
+        "eval/**/*.py",
+        "evals/**/*.py",
+        "src/**/*.py",
+        "tests/**/*.py",
+        "spec/**/*.py",
+    ]
+
+    found: list[str] = []
+    for pattern in patterns:
+        for f in root.glob(pattern):
+            if f.name.startswith("_"):
+                continue
+            try:
+                content = f.read_text(encoding="utf-8", errors="ignore")
+                if "define_eval" in content:
+                    found.append(str(f))
+            except OSError:
+                continue
+    return found
+
+
+def find_legacy_config(project_root: str) -> str | None:
+    """Search for legacy config files."""
+    root = Path(project_root)
+    candidates = [
+        "evalai.config.json",
+        "evalai.config.py",
+        "evalgate.config.json",
+        ".evalgaterc",
+        ".evalgaterc.json",
+        ".evalgate/config.json",
+        ".evalai/config.json",
+    ]
+    for name in candidates:
+        p = root / name
+        if p.exists():
+            return str(p)
+    return None
+
+
+def get_execution_mode(project_root: str | None = None) -> ExecutionModeConfig:
+    """Determine execution mode from environment or auto-detection."""
+    root = project_root or os.getcwd()
+    env_mode = os.environ.get(ENV_VAR, "").lower()
+
+    if env_mode in POSSIBLE_VALUES:
+        spec_files = find_spec_files(root) if env_mode != "legacy" else []
+        legacy_config = find_legacy_config(root) if env_mode != "spec" else None
+        return ExecutionModeConfig(
+            mode=env_mode,  # type: ignore[arg-type]
+            has_spec_runtime=env_mode != "legacy",
+            has_legacy_runtime=env_mode != "spec",
+            project_root=root,
+            spec_files=spec_files,
+            legacy_config=legacy_config,
+        )
+
+    # Auto-detect
+    spec_files = find_spec_files(root)
+    legacy_config = find_legacy_config(root)
+    has_spec = len(spec_files) > 0
+    has_legacy = legacy_config is not None
+
+    if has_spec:
+        mode: ExecutionMode = "spec"
+    elif has_legacy:
+        mode = "legacy"
+    else:
+        mode = "auto"
+
+    return ExecutionModeConfig(
+        mode=mode,
+        has_spec_runtime=has_spec,
+        has_legacy_runtime=has_legacy,
+        project_root=root,
+        spec_files=spec_files,
+        legacy_config=legacy_config,
+    )
+
+
+def can_run_spec_mode(config: ExecutionModeConfig) -> bool:
+    return config.has_spec_runtime and len(config.spec_files) > 0
+
+
+def can_run_legacy_mode(config: ExecutionModeConfig) -> bool:
+    return config.has_legacy_runtime and config.legacy_config is not None
+
+
+def get_recommended_mode(config: ExecutionModeConfig) -> ExecutionMode:
+    if config.mode != "auto":
+        return config.mode
+    if can_run_spec_mode(config):
+        return "spec"
+    if can_run_legacy_mode(config):
+        return "legacy"
+    return "auto"
+
+
+def validate_execution_mode(config: ExecutionModeConfig) -> dict[str, list[str] | bool]:
+    """Validate execution mode compatibility."""
+    warnings: list[str] = []
+    errors: list[str] = []
+
+    if config.has_spec_runtime and config.has_legacy_runtime:
+        warnings.append(
+            "Project contains both spec files and legacy config. Consider migrating legacy tests to spec format."
+        )
+
+    if not config.has_spec_runtime and not config.has_legacy_runtime:
+        warnings.append("No tests found. Use 'evalgate init' to create a new project.")
+
+    if config.mode == "spec" and not can_run_spec_mode(config):
+        errors.append(
+            "Spec mode requested but no spec files found. Create spec files with define_eval() or use legacy mode."
+        )
+
+    if config.mode == "legacy" and not can_run_legacy_mode(config):
+        errors.append("Legacy mode requested but no config file found. Create a config file or use spec mode.")
+
+    return {"valid": len(errors) == 0, "warnings": warnings, "errors": errors}
+
+
+def has_execution_mode_env() -> bool:
+    return ENV_VAR in os.environ
+
+
+def get_execution_mode_env() -> str | None:
+    return os.environ.get(ENV_VAR)
+
+
+def set_execution_mode_env(mode: ExecutionMode) -> None:
+    os.environ[ENV_VAR] = mode
+
+
+def clear_execution_mode_env() -> None:
+    os.environ.pop(ENV_VAR, None)

evalgate_sdk/runtime/executor.py

@@ -0,0 +1,92 @@
+"""Local executor — runs eval specs with timeout protection."""
+
+from __future__ import annotations
+
+import asyncio
+import time
+
+from evalgate_sdk.runtime.types import (
+    EvalContext,
+    EvalResult,
+    EvalSpec,
+    ExecutorCapabilities,
+)
+
+
+class LocalExecutor:
+    """Execute eval specs locally with timeout and error handling."""
+
+    def __init__(self) -> None:
+        self.capabilities = ExecutorCapabilities(
+            supports_async=True,
+            supports_timeout=True,
+            supports_retries=True,
+            supports_parallel=False,
+        )
+
+    async def execute(self, spec: EvalSpec, context: EvalContext) -> EvalResult:
+        timeout_s = spec.options.timeout_ms / 1000
+        start = time.monotonic()
+
+        for attempt in range(1 + spec.options.retries):
+            try:
+                result = spec.executor(context)
+                if asyncio.iscoroutine(result) or asyncio.isfuture(result) or hasattr(result, "__await__"):
+                    result = await asyncio.wait_for(result, timeout=timeout_s)
+
+                duration = (time.monotonic() - start) * 1000
+
+                if isinstance(result, EvalResult):
+                    result.duration_ms = duration
+                    return result
+
+                if isinstance(result, dict):
+                    return EvalResult(
+                        passed=result.get("passed", result.get("pass", False)),
+                        score=result.get("score", 1.0 if result.get("passed", result.get("pass")) else 0.0),
+                        assertions=result.get("assertions", []),
+                        metadata=result.get("metadata", {}),
+                        error=result.get("error"),
+                        duration_ms=duration,
+                        status="passed" if result.get("passed", result.get("pass")) else "failed",
+                    )
+
+                passed = bool(result)
+                return EvalResult(
+                    passed=passed,
+                    score=1.0 if passed else 0.0,
+                    duration_ms=duration,
+                    status="passed" if passed else "failed",
+                )
+
+            except asyncio.TimeoutError:
+                duration = (time.monotonic() - start) * 1000
+                if attempt < spec.options.retries:
+                    continue
+                return EvalResult(
+                    passed=False,
+                    score=0.0,
+                    duration_ms=duration,
+                    error=f"Timeout after {spec.options.timeout_ms}ms",
+                    status="timeout",
+                )
+            except Exception as exc:
+                duration = (time.monotonic() - start) * 1000
+                if attempt < spec.options.retries:
+                    continue
+                return EvalResult(
+                    passed=False,
+                    score=0.0,
+                    duration_ms=duration,
+                    error=str(exc),
+                    status="error",
+                )
+
+        return EvalResult(passed=False, score=0.0, error="Max retries exceeded", status="error")
+
+
+def create_local_executor() -> LocalExecutor:
+    return LocalExecutor()
+
+
+default_local_executor = LocalExecutor()

evalgate_sdk/runtime/registry.py

@@ -0,0 +1,125 @@
+"""Runtime registry — manages eval spec lifecycle and lookup."""
+
+from __future__ import annotations
+
+import builtins
+import hashlib
+import time
+from collections.abc import Callable
+from typing import Any, TypeVar
+
+from evalgate_sdk.runtime.types import (
+    EvalSpec,
+    RuntimeHealth,
+    SpecRegistrationError,
+)
+
+T = TypeVar("T")
+
+
+class EvalRuntime:
+    """In-process registry for eval specs."""
+
+    def __init__(self, namespace: str = "default") -> None:
+        self._namespace = namespace
+        self._specs: dict[str, EvalSpec] = {}
+        self._started_at = time.monotonic()
+
+    @property
+    def namespace(self) -> str:
+        return self._namespace
+
+    def register(self, spec: EvalSpec) -> None:
+        if spec.id in self._specs:
+            raise SpecRegistrationError(f"Spec '{spec.id}' already registered")
+        self._specs[spec.id] = spec
+
+    def get(self, spec_id: str) -> EvalSpec | None:
+        return self._specs.get(spec_id)
+
+    def list(self, *, suite: str | None = None, tags: builtins.list[str] | None = None) -> builtins.list[EvalSpec]:
+        specs = list(self._specs.values())
+        if suite is not None:
+            specs = [s for s in specs if s.suite == suite]
+        if tags:
+            tag_set = set(tags)
+            specs = [s for s in specs if tag_set.issubset(set(s.options.tags))]
+        return specs
+
+    def find(self, pattern: str) -> builtins.list[EvalSpec]:
+        return [s for s in self._specs.values() if pattern in s.name or pattern in s.id]
+
+    def clear(self) -> None:
+        self._specs.clear()
+
+    def get_health(self) -> RuntimeHealth:
+        return RuntimeHealth(
+            status="healthy",
+            spec_count=len(self._specs),
+            memory_estimate_mb=len(self._specs) * 0.001,
+            uptime_ms=(time.monotonic() - self._started_at) * 1000,
+        )
+
+
+class RuntimeHandle:
+    """Scoped runtime with lifecycle management."""
+
+    def __init__(self, runtime: EvalRuntime) -> None:
+        self.runtime = runtime
+
+    def define_eval(self, spec: EvalSpec) -> None:
+        self.runtime.register(spec)
+
+    def dispose(self) -> None:
+        self.runtime.clear()
+
+    def snapshot(self) -> dict[str, Any]:
+        return {
+            "namespace": self.runtime.namespace,
+            "specs": [
+                {"id": s.id, "name": s.name, "suite": s.suite, "tags": s.options.tags} for s in self.runtime.list()
+            ],
+        }
+
+    def load(self, data: dict[str, Any]) -> None:
+        pass
+
+
+_active_runtime: EvalRuntime | None = None
+
+
+def create_eval_runtime(project_root: str | None = None) -> RuntimeHandle:
+    namespace = "default"
+    if project_root:
+        namespace = hashlib.sha256(project_root.encode()).hexdigest()[:12]
+    runtime = EvalRuntime(namespace=namespace)
+    global _active_runtime
+    _active_runtime = runtime
+    return RuntimeHandle(runtime)
+
+
+def get_active_runtime() -> EvalRuntime | None:
+    return _active_runtime
+
+
+def set_active_runtime(runtime: EvalRuntime) -> None:
+    global _active_runtime
+    _active_runtime = runtime
+
+
+def dispose_active_runtime() -> None:
+    global _active_runtime
+    if _active_runtime:
+        _active_runtime.clear()
+        _active_runtime = None
+
+
+async def with_runtime(project_root: str, fn: Callable[[RuntimeHandle], Any]) -> Any:
+    handle = create_eval_runtime(project_root)
+    try:
+        result = fn(handle)
+        if hasattr(result, "__await__"):
+            return await result
+        return result
+    finally:
+        handle.dispose()