evalgate-sdk 3.3.1__py3-none-any.whl

evalgate_sdk/logger.py

@@ -0,0 +1,123 @@
+"""Structured logger with levels, child loggers, and request/response helpers."""
+
+from __future__ import annotations
+
+import logging
+import sys
+import time
+from collections.abc import Callable
+from typing import Any, Literal
+
+LogLevel = Literal["trace", "debug", "info", "warn", "error"]
+
+_LEVEL_MAP: dict[str, int] = {
+    "trace": 5,
+    "debug": logging.DEBUG,
+    "info": logging.INFO,
+    "warn": logging.WARNING,
+    "error": logging.ERROR,
+}
+
+
+class Logger:
+    """SDK logger with structured output, child loggers, and request tracing."""
+
+    def __init__(
+        self,
+        *,
+        level: LogLevel = "info",
+        prefix: str = "evalai",
+        handler: Callable[[str, str, Any], None] | None = None,
+    ) -> None:
+        self._level = level
+        self._prefix = prefix
+        self._handler = handler
+        self._py_logger = logging.getLogger(f"evalgate_sdk.{prefix}")
+        self._py_logger.setLevel(_LEVEL_MAP.get(level, logging.INFO))
+        if not self._py_logger.handlers:
+            h = logging.StreamHandler(sys.stderr)
+            h.setFormatter(logging.Formatter("%(asctime)s [%(name)s] %(levelname)s %(message)s"))
+            self._py_logger.addHandler(h)
+
+    def set_level(self, level: LogLevel) -> None:
+        self._level = level
+        self._py_logger.setLevel(_LEVEL_MAP.get(level, logging.INFO))
+
+    def is_level_enabled(self, level: LogLevel) -> bool:
+        return _LEVEL_MAP.get(level, 0) >= _LEVEL_MAP.get(self._level, 0)
+
+    def _emit(self, level: LogLevel, message: str, data: Any = None) -> None:
+        if self._handler:
+            self._handler(level, message, data)
+            return
+        extra = f" {data}" if data is not None else ""
+        py_level = _LEVEL_MAP.get(level, logging.INFO)
+        self._py_logger.log(py_level, "%s%s", message, extra)
+
+    def trace(self, message: str, data: Any = None) -> None:
+        if self.is_level_enabled("trace"):
+            self._emit("trace", message, data)
+
+    def debug(self, message: str, data: Any = None) -> None:
+        if self.is_level_enabled("debug"):
+            self._emit("debug", message, data)
+
+    def info(self, message: str, data: Any = None) -> None:
+        if self.is_level_enabled("info"):
+            self._emit("info", message, data)
+
+    def warn(self, message: str, data: Any = None) -> None:
+        if self.is_level_enabled("warn"):
+            self._emit("warn", message, data)
+
+    def error(self, message: str, data: Any = None) -> None:
+        if self.is_level_enabled("error"):
+            self._emit("error", message, data)
+
+    def log_request(self, method: str, url: str, data: Any = None) -> None:
+        self.debug(f"→ {method} {url}", data)
+
+    def log_response(self, method: str, url: str, status: int, duration_ms: float, data: Any = None) -> None:
+        self.debug(f"← {method} {url} {status} ({duration_ms:.0f}ms)", data)
+
+    def child(self, prefix: str) -> Logger:
+        return Logger(
+            level=self._level,
+            prefix=f"{self._prefix}:{prefix}",
+            handler=self._handler,
+        )
+
+
+class RequestLogger:
+    """Convenience wrapper that times request/response pairs."""
+
+    def __init__(self, logger: Logger) -> None:
+        self._logger = logger
+        self._start: float = 0
+
+    def on_request(self, method: str, url: str, body: Any = None) -> None:
+        self._start = time.monotonic()
+        self._logger.log_request(method, url, body)
+
+    def on_response(self, method: str, url: str, status: int, body: Any = None) -> None:
+        elapsed = (time.monotonic() - self._start) * 1000
+        self._logger.log_response(method, url, status, elapsed, body)
+
+
+_global_logger: Logger | None = None
+
+
+def create_logger(level: LogLevel = "info", **kwargs: Any) -> Logger:
+    return Logger(level=level, **kwargs)
+
+
+def get_logger() -> Logger:
+    global _global_logger
+    if _global_logger is None:
+        _global_logger = create_logger()
+    return _global_logger
+
+
+def set_logger(logger: Logger) -> None:
+    global _global_logger
+    _global_logger = logger

evalgate_sdk/matchers.py

@@ -0,0 +1,62 @@
+"""Pytest plugin — provides ``to_pass_gate`` assertion for regression results."""
+
+from __future__ import annotations
+
+from typing import Any
+
+
+def to_pass_gate(result: Any) -> bool:
+    """Check if an eval result passes the gate.
+
+    Works with ``OpenAIChatEvalResult`` and any object with a ``passed`` attribute.
+
+    Usage in pytest::
+
+        from evalgate_sdk.matchers import to_pass_gate
+
+        result = await openai_chat_eval(...)
+        assert to_pass_gate(result)
+    """
+    if hasattr(result, "passed"):
+        return bool(result.passed)
+    if isinstance(result, dict):
+        return bool(result.get("passed", False))
+    return False
+
+
+class GateAssertionError(AssertionError):
+    """Raised when a gate assertion fails with diagnostic info."""
+
+    def __init__(self, result: Any) -> None:
+        self.result = result
+        score = getattr(result, "score", "?")
+        total = getattr(result, "total", "?")
+        passed = getattr(result, "passed_count", "?")
+        super().__init__(f"Gate assertion failed: {passed}/{total} passed (score={score})")
+
+
+def assert_passes_gate(result: Any, message: str = "") -> None:
+    """Assert that a result passes the gate, with rich error output.
+
+    Args:
+        result: The eval result to check (must have a ``passed`` attribute or key).
+        message: Optional custom error message to display on failure.
+    """
+    if not to_pass_gate(result):
+        if message:
+            raise AssertionError(message)
+        raise GateAssertionError(result)
+
+
+# ── Pytest plugin ────────────────────────────────────────────────────
+
+try:
+    import pytest
+
+    @pytest.fixture
+    def gate_result():
+        """Fixture that provides a gate assertion helper."""
+        return assert_passes_gate
+
+except ImportError:
+    pass

evalgate_sdk/otel.py

@@ -0,0 +1,256 @@
+"""OpenTelemetry exporter for WorkflowTracer spans (T6).
+
+Port of the TypeScript SDK's ``otel.ts``.
+Converts workflow tracer data into OTLP-compatible JSON payloads.
+"""
+
+from __future__ import annotations
+
+import os
+import random
+import time
+from dataclasses import dataclass, field
+from typing import Any
+
+import httpx
+
+
+def _generate_trace_id() -> str:
+    """Generate a 32-hex-char trace ID."""
+    return f"{random.getrandbits(128):032x}"
+
+
+def _generate_span_id() -> str:
+    """Generate a 16-hex-char span ID."""
+    return f"{random.getrandbits(64):016x}"
+
+
+def _ms_to_ns(ms: float) -> int:
+    """Convert milliseconds to nanoseconds."""
+    return int(ms * 1_000_000)
+
+
+@dataclass
+class OTelAttribute:
+    key: str
+    value: Any
+
+    def to_dict(self) -> dict[str, Any]:
+        if isinstance(self.value, bool):
+            return {"key": self.key, "value": {"boolValue": self.value}}
+        if isinstance(self.value, int):
+            return {"key": self.key, "value": {"intValue": str(self.value)}}
+        if isinstance(self.value, float):
+            return {"key": self.key, "value": {"doubleValue": self.value}}
+        return {"key": self.key, "value": {"stringValue": str(self.value)}}
+
+
+@dataclass
+class OTelEvent:
+    name: str
+    time_unix_nano: int = 0
+    attributes: list[OTelAttribute] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {
+            "name": self.name,
+            "timeUnixNano": str(self.time_unix_nano),
+            "attributes": [a.to_dict() for a in self.attributes],
+        }
+
+
+@dataclass
+class OTelSpan:
+    trace_id: str
+    span_id: str
+    name: str
+    start_time_unix_nano: int
+    end_time_unix_nano: int
+    parent_span_id: str | None = None
+    kind: int = 1  # SPAN_KIND_INTERNAL
+    status_code: int = 1  # STATUS_CODE_OK
+    status_message: str = ""
+    attributes: list[OTelAttribute] = field(default_factory=list)
+    events: list[OTelEvent] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        d: dict[str, Any] = {
+            "traceId": self.trace_id,
+            "spanId": self.span_id,
+            "name": self.name,
+            "kind": self.kind,
+            "startTimeUnixNano": str(self.start_time_unix_nano),
+            "endTimeUnixNano": str(self.end_time_unix_nano),
+            "status": {"code": self.status_code, "message": self.status_message},
+            "attributes": [a.to_dict() for a in self.attributes],
+            "events": [e.to_dict() for e in self.events],
+        }
+        if self.parent_span_id:
+            d["parentSpanId"] = self.parent_span_id
+        return d
+
+
+@dataclass
+class OTelExportPayload:
+    """OTLP JSON export payload."""
+
+    resource_spans: list[dict[str, Any]] = field(default_factory=list)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"resourceSpans": self.resource_spans}
+
+
+@dataclass
+class OTelExporterOptions:
+    endpoint: str = ""
+    service_name: str = "evalgate-sdk"
+    headers: dict[str, str] = field(default_factory=dict)
+    timeout_ms: int = 10_000
+
+
+class OTelExporter:
+    """Exports evaluation data as OpenTelemetry spans."""
+
+    def __init__(self, options: OTelExporterOptions | None = None) -> None:
+        opts = options or OTelExporterOptions()
+        self._endpoint = opts.endpoint or os.environ.get("OTEL_EXPORTER_OTLP_ENDPOINT", "http://localhost:4318")
+        self._service_name = opts.service_name
+        self._headers = opts.headers
+        self._timeout = opts.timeout_ms / 1000.0
+
+    def export_from_tracer(self, tracer: Any) -> OTelExportPayload:
+        """Convert a WorkflowTracer into an OTLP payload.
+
+        *tracer* should expose ``.spans`` (list of dicts with name, start_time,
+        end_time, metadata, parent_span_id, etc.).
+        """
+        trace_id = _generate_trace_id()
+        spans: list[OTelSpan] = []
+
+        raw_spans = getattr(tracer, "spans", [])
+        for raw in raw_spans:
+            span_id = _generate_span_id()
+            attrs: list[OTelAttribute] = []
+
+            if isinstance(raw, dict):
+                name = raw.get("name", "unknown")
+                start_ns = _ms_to_ns(raw.get("start_time", 0))
+                end_ns = _ms_to_ns(raw.get("end_time", start_ns))
+                parent = raw.get("parent_span_id")
+                for k, v in raw.get("metadata", {}).items():
+                    attrs.append(OTelAttribute(key=f"evalgate.{k}", value=v))
+            else:
+                name = getattr(raw, "name", "unknown")
+                start_ns = _ms_to_ns(getattr(raw, "start_time", 0))
+                end_ns = _ms_to_ns(getattr(raw, "end_time", start_ns))
+                parent = getattr(raw, "parent_span_id", None)
+
+            attrs.append(OTelAttribute(key="evalgate.service", value=self._service_name))
+            spans.append(
+                OTelSpan(
+                    trace_id=trace_id,
+                    span_id=span_id,
+                    name=name,
+                    start_time_unix_nano=start_ns,
+                    end_time_unix_nano=end_ns,
+                    parent_span_id=parent,
+                    attributes=attrs,
+                )
+            )
+
+        return self._build_payload(spans)
+
+    def export_run_result(
+        self,
+        run_id: str,
+        results: list[dict[str, Any]],
+        start_time_ms: float | None = None,
+        end_time_ms: float | None = None,
+    ) -> OTelExportPayload:
+        """Convert evaluation run results into an OTLP payload."""
+        trace_id = _generate_trace_id()
+        now_ms = time.time() * 1000
+        root_start = _ms_to_ns(start_time_ms or now_ms)
+        root_end = _ms_to_ns(end_time_ms or now_ms)
+
+        root_span_id = _generate_span_id()
+        spans: list[OTelSpan] = [
+            OTelSpan(
+                trace_id=trace_id,
+                span_id=root_span_id,
+                name=f"evalgate.run.{run_id}",
+                start_time_unix_nano=root_start,
+                end_time_unix_nano=root_end,
+                attributes=[
+                    OTelAttribute(key="evalgate.run_id", value=run_id),
+                    OTelAttribute(key="evalgate.service", value=self._service_name),
+                ],
+            )
+        ]
+
+        for r in results:
+            span_id = _generate_span_id()
+            duration_ms = r.get("duration_ms", 0)
+            s_start = _ms_to_ns(r.get("start_time_ms", now_ms))
+            s_end = s_start + _ms_to_ns(duration_ms)
+
+            status = 1 if r.get("passed") else 2  # OK or ERROR
+            attrs = [
+                OTelAttribute(key="evalgate.test_id", value=r.get("test_id", "")),
+                OTelAttribute(key="evalgate.test_name", value=r.get("test_name", "")),
+                OTelAttribute(key="evalgate.score", value=r.get("score", 0)),
+                OTelAttribute(key="evalgate.passed", value=r.get("passed", False)),
+            ]
+            spans.append(
+                OTelSpan(
+                    trace_id=trace_id,
+                    span_id=span_id,
+                    parent_span_id=root_span_id,
+                    name=f"evalgate.spec.{r.get('test_name', 'unknown')}",
+                    start_time_unix_nano=s_start,
+                    end_time_unix_nano=s_end,
+                    status_code=status,
+                    status_message=r.get("error", ""),
+                    attributes=attrs,
+                )
+            )
+
+        return self._build_payload(spans)
+
+    def _build_payload(self, spans: list[OTelSpan]) -> OTelExportPayload:
+        resource = {
+            "resource": {
+                "attributes": [
+                    OTelAttribute(key="service.name", value=self._service_name).to_dict(),
+                ],
+            },
+            "scopeSpans": [
+                {
+                    "scope": {"name": "evalgate-sdk"},
+                    "spans": [s.to_dict() for s in spans],
+                }
+            ],
+        }
+        return OTelExportPayload(resource_spans=[resource])
+
+    async def send(self, payload: OTelExportPayload) -> bool:
+        """POST the OTLP payload to the collector endpoint.
+
+        Returns False on connection/timeout errors instead of raising.
+        """
+        url = f"{self._endpoint.rstrip('/')}/v1/traces"
+        try:
+            async with httpx.AsyncClient(timeout=self._timeout) as client:
+                resp = await client.post(
+                    url,
+                    json=payload.to_dict(),
+                    headers={"Content-Type": "application/json", **self._headers},
+                )
+                return resp.status_code < 400
+        except (httpx.ConnectError, httpx.TimeoutException, OSError):
+            return False
+
+
+def create_otel_exporter(options: OTelExporterOptions | None = None) -> OTelExporter:
+    """Factory function for OTelExporter."""
+    return OTelExporter(options)

evalgate_sdk/pagination.py

@@ -0,0 +1,145 @@
+"""Cursor-based pagination helpers — iterators, auto-pagination, and encoding."""
+
+from __future__ import annotations
+
+import base64
+import json
+from collections.abc import AsyncIterator, Callable
+from typing import Any, Generic, TypeVar
+
+T = TypeVar("T")
+
+
+def encode_cursor(data: Any) -> str:
+    """Encode arbitrary data as a base64 cursor string."""
+    return base64.urlsafe_b64encode(json.dumps(data).encode()).decode()
+
+
+def decode_cursor(cursor: str) -> Any:
+    """Decode a base64 cursor string back to its original value."""
+    return json.loads(base64.urlsafe_b64decode(cursor.encode()).decode())
+
+
+class PaginatedResponse(Generic[T]):
+    """Container for a page of results plus pagination metadata."""
+
+    def __init__(self, data: list[T], has_more: bool, total: int | None = None) -> None:
+        self.data = data
+        self.has_more = has_more
+        self.total = total
+
+
+class PaginatedIterator(Generic[T]):
+    """Async iterator that automatically fetches pages.
+
+    Usage::
+
+        pages = PaginatedIterator(fetch_fn, limit=20)
+        async for page in pages:
+            for item in page:
+                print(item)
+
+        # or collect everything
+        all_items = await pages.to_list()
+    """
+
+    def __init__(
+        self,
+        fetch_fn: Callable[[int, int], Any],
+        limit: int = 20,
+    ) -> None:
+        self._fetch_fn = fetch_fn
+        self._limit = limit
+        self._offset = 0
+        self._exhausted = False
+
+    def __aiter__(self) -> PaginatedIterator[T]:
+        return self
+
+    async def __anext__(self) -> list[T]:
+        if self._exhausted:
+            raise StopAsyncIteration
+
+        result = await self._fetch_fn(self._offset, self._limit)
+
+        if isinstance(result, dict):
+            data = result.get("data", [])
+            has_more = result.get("has_more", result.get("hasMore", False))
+        elif isinstance(result, PaginatedResponse):
+            data = result.data
+            has_more = result.has_more
+        elif isinstance(result, list):
+            data = result
+            has_more = len(data) >= self._limit
+        else:
+            data = list(result)
+            has_more = len(data) >= self._limit
+
+        if not data:
+            self._exhausted = True
+            raise StopAsyncIteration
+
+        self._offset += len(data)
+        if not has_more:
+            self._exhausted = True
+
+        return data
+
+    async def to_list(self) -> list[T]:
+        """Collect all pages into a single flat list."""
+        items: list[T] = []
+        async for page in self:
+            items.extend(page)
+        return items
+
+    def reset(self) -> None:
+        self._offset = 0
+        self._exhausted = False
+
+
+def create_paginated_iterator(
+    fetch_fn: Callable[[int, int], Any],
+    limit: int = 20,
+) -> PaginatedIterator[Any]:
+    """Create a paginated iterator from a fetch function."""
+    return PaginatedIterator(fetch_fn, limit)
+
+
+async def auto_paginate(
+    fetch_fn: Callable[[int, int], Any],
+    limit: int = 20,
+) -> AsyncIterator[Any]:
+    """Auto-paginate and yield individual items."""
+    iterator = PaginatedIterator(fetch_fn, limit)
+    async for page in iterator:
+        for item in page:
+            yield item
+
+
+def create_pagination_meta(
+    items: list[Any],
+    limit: int,
+    offset: int,
+    total: int | None = None,
+) -> dict[str, Any]:
+    """Create pagination metadata for an API response."""
+    return {
+        "limit": limit,
+        "offset": offset,
+        "count": len(items),
+        "total": total,
+        "has_more": total is not None and (offset + len(items)) < total if total else len(items) >= limit,
+    }
+
+
+def parse_pagination_params(
+    params: dict[str, Any] | None = None,
+    default_limit: int = 20,
+    max_limit: int = 100,
+) -> dict[str, int]:
+    """Parse and clamp pagination params."""
+    if params is None:
+        return {"limit": default_limit, "offset": 0}
+    limit = min(int(params.get("limit", default_limit)), max_limit)
+    offset = max(int(params.get("offset", 0)), 0)
+    return {"limit": limit, "offset": offset}

evalgate_sdk/pytest_plugin.py

@@ -0,0 +1,96 @@
+"""Pytest plugin for EvalGate assertions (T9).
+
+Provides custom pytest assertions for eval results and a plugin
+that can be auto-registered via pyproject.toml entry points.
+
+Usage::
+
+    from evalgate_sdk.pytest_plugin import assert_passes_gate, assert_score_above
+
+    def test_chatbot_quality(eval_result):
+        assert_passes_gate(eval_result)
+        assert_score_above(eval_result, 90.0)
+
+Note: pytest is imported lazily so the SDK can be used without pytest installed.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING, Any
+
+if TYPE_CHECKING:
+    import pytest as _pytest_type  # noqa: F401
+
+
+def _get_pytest() -> Any:
+    """Lazily import pytest, raising a clear error if not installed."""
+    try:
+        import pytest
+
+        return pytest
+    except ImportError as exc:
+        raise ImportError(
+            "pytest is required for evalgate_sdk.pytest_plugin. Install with: pip install pytest"
+        ) from exc
+
+
+def assert_passes_gate(result: Any, message: str = "") -> None:
+    """Assert that an eval result passes the quality gate.
+
+    *result* should have a ``passed`` attribute (or key) that is truthy.
+
+    This delegates to :func:`evalgate_sdk.matchers.assert_passes_gate` for consistency.
+    """
+    from evalgate_sdk.matchers import assert_passes_gate as _assert_passes_gate
+
+    _assert_passes_gate(result, message)
+
+
+def assert_score_above(result: Any, threshold: float, message: str = "") -> None:
+    """Assert that the eval result score is above *threshold*."""
+    pytest = _get_pytest()
+    score = _get_field(result, "score")
+    if score is None:
+        pytest.fail(message or "Result has no 'score' field")
+    if score < threshold:
+        pytest.fail(message or f"Score {score} is below threshold {threshold}")
+
+
+def assert_score_between(result: Any, min_score: float, max_score: float, message: str = "") -> None:
+    """Assert that the eval result score is within [min_score, max_score]."""
+    pytest = _get_pytest()
+    score = _get_field(result, "score")
+    if score is None:
+        pytest.fail(message or "Result has no 'score' field")
+    if score < min_score or score > max_score:
+        pytest.fail(message or f"Score {score} not in [{min_score}, {max_score}]")
+
+
+def assert_no_errors(result: Any, message: str = "") -> None:
+    """Assert that the eval result has no errors."""
+    pytest = _get_pytest()
+    error = _get_field(result, "error")
+    status = _get_field(result, "status")
+    if error:
+        pytest.fail(message or f"Eval result has error: {error}")
+    if status == "error":
+        pytest.fail(message or "Eval result status is 'error'")
+
+
+def assert_all_assertions_passed(result: Any, message: str = "") -> None:
+    """Assert that all sub-assertions in the result passed."""
+    pytest = _get_pytest()
+    assertions = _get_field(result, "assertions") or []
+    for i, assertion in enumerate(assertions):
+        passed = _get_field(assertion, "passed")
+        if not passed:
+            name = _get_field(assertion, "assertion_type") or _get_field(assertion, "name") or f"assertion-{i}"
+            msg = _get_field(assertion, "message") or "failed"
+            pytest.fail(message or f"Sub-assertion '{name}' failed: {msg}")
+
+
+def _get_field(obj: Any, field: str) -> Any:
+    """Get a field from a dict or dataclass/object."""
+    if isinstance(obj, dict):
+        return obj.get(field)
+    return getattr(obj, field, None)