aimeval 0.6.0__py3-none-any.whl

aimeval/__init__.py

@@ -0,0 +1,191 @@
+"""
+AIMEval Python SDK — programmatic access for CI/CD pipelines.
+
+Quick start::
+
+    from aimeval import AIMEval
+
+    client = AIMEval(api_key="aime_sk_...", base_url="https://app.aimeval.com")
+
+    # End-to-end workflow with a live progress bar (TTY only):
+    run = client.evaluate(
+        name="nightly-regression",
+        model=MODEL_ID,
+        dataset=DATASET_ID,
+        metrics=COLLECTION_ID,
+    )
+    sys.exit(0 if run.passed else 1)
+
+    # Or step by step (industry-standard resource namespaces):
+    run = client.runs.create(name="...", model=..., dataset=..., metrics=...)
+    run = client.runs.wait(run.id, progress=True)
+
+    # Auto-pagination — iterate every page transparently:
+    for run in client.runs.list(status="completed"):
+        print(run.id, run.score)
+"""
+from aimeval.client import AIMEval, AsyncAIMEval
+from aimeval._otel import disable_otel, enable_otel, is_available as otel_is_available, otel_enabled
+from aimeval import events
+from aimeval._trace import Span, TraceCollector, records, span, trace
+from aimeval._trace_io import (
+    ReplayResult,
+    load_trace,
+    load_trace_iter,
+    replay_trace,
+    save_trace,
+)
+from aimeval._webhook_verify import (
+    WebhookVerificationError,
+    construct_event,
+    verify_webhook,
+)
+from aimeval._eval import (
+    Eval,
+    EvalAssertionError,
+    EvalResult,
+    EvalScore,
+    assert_test,
+    eval_from_run,
+)
+from aimeval import metrics
+from aimeval._exceptions import (
+    AIMEvalError,
+    AIMEvalSecurityWarning,
+    APIConnectionError,
+    APIServerError,
+    APITimeoutError,
+    AuthenticationError,
+    BadRequestError,
+    ConflictError,
+    NotFoundError,
+    RateLimitError,
+)
+from aimeval._streaming import (
+    AsyncRunSSEStream,
+    AsyncRunStream,
+    RunEvent,
+    RunSSEStream,
+    RunStream,
+)
+from aimeval._types import (
+    Annotation,
+    AnnotationsBootstrapResult,
+    AuditEntry,
+    CompareHistoryItem,
+    Comparison,
+    ConnectionTestResult,
+    CostEstimate,
+    Dataset,
+    GateCompatibility,
+    MetricCollection,
+    MetricDistribution,
+    MetricRegistry,
+    MetricRegistryEntry,
+    MetricSummary,
+    Model,
+    Prompt,
+    PromptDiff,
+    PromptTestResult,
+    PromptTestRun,
+    PromptVersion,
+    QualityGate,
+    RegressionSet,
+    Resource,
+    Run,
+    SearchHit,
+    SearchResults,
+    UploadStatus,
+    UploadTicket,
+    Usage,
+    Webhook,
+    WizardBootstrap,
+)
+
+
+__version__ = "0.6.0"
+
+__all__ = [
+    # Clients
+    "AIMEval",
+    "AsyncAIMEval",
+    "__version__",
+    # Tracing (local span capture → eval datasets; optional OTel mirror)
+    "trace",
+    "span",
+    "TraceCollector",
+    "Span",
+    "records",
+    "events",
+    # Trace recording + replay (prod-to-eval pattern)
+    "save_trace",
+    "load_trace",
+    "load_trace_iter",
+    "replay_trace",
+    "ReplayResult",
+    "enable_otel",
+    "disable_otel",
+    "otel_enabled",
+    "otel_is_available",
+    # Eval harness + metrics (CI assertion surface)
+    "Eval",
+    "eval_from_run",
+    "assert_test",
+    "EvalResult",
+    "EvalScore",
+    "EvalAssertionError",
+    "metrics",
+    # Errors
+    "AIMEvalError",
+    "AIMEvalSecurityWarning",
+    "APIConnectionError",
+    "APIServerError",
+    "APITimeoutError",
+    "AuthenticationError",
+    "BadRequestError",
+    "ConflictError",
+    "NotFoundError",
+    "RateLimitError",
+    # Response types
+    "Annotation",
+    "AnnotationsBootstrapResult",
+    "AuditEntry",
+    "CompareHistoryItem",
+    "Comparison",
+    "ConnectionTestResult",
+    "CostEstimate",
+    "Dataset",
+    "GateCompatibility",
+    "MetricCollection",
+    "MetricDistribution",
+    "MetricRegistry",
+    "MetricRegistryEntry",
+    "MetricSummary",
+    "Model",
+    "Prompt",
+    "PromptDiff",
+    "PromptTestResult",
+    "PromptTestRun",
+    "PromptVersion",
+    "QualityGate",
+    "RegressionSet",
+    "Resource",
+    "Run",
+    "SearchHit",
+    "SearchResults",
+    "UploadStatus",
+    "UploadTicket",
+    "Usage",
+    "Webhook",
+    "WizardBootstrap",
+    # Webhook verifier (security helper)
+    "WebhookVerificationError",
+    "construct_event",
+    "verify_webhook",
+    # Streaming
+    "AsyncRunSSEStream",
+    "AsyncRunStream",
+    "RunEvent",
+    "RunSSEStream",
+    "RunStream",
+]

aimeval/_beta.py

@@ -0,0 +1,61 @@
+"""``client.beta.*`` — experimental surface area.
+
+Industry pattern from OpenAI / Anthropic: features that haven't earned
+SemVer protection yet live under a dedicated ``beta`` namespace. This
+gives them three properties at once:
+
+  1. **Visible**: a glance at ``client.beta.<something>`` tells the
+     reader "this isn't stable yet — pin your SDK and read the
+     CHANGELOG before bumping".
+  2. **Safe to remove / rename** without bumping the major version of
+     the SDK. The ``beta`` package itself is the contract.
+  3. **Cheap to ship**: graduation to the stable surface (e.g.
+     ``client.replay``) is a one-line re-export when the API settles.
+
+Currently in beta:
+
+  - :attr:`Beta.replay` — trace recording + replay helpers
+    (:func:`aimeval.save_trace` / :func:`aimeval.load_trace` /
+    :func:`aimeval.replay_trace`). Wrapped under ``client.beta.replay``
+    so the helpers feel discoverable from autocomplete even when
+    callers don't import them directly.
+"""
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+from aimeval import _trace_io
+
+if TYPE_CHECKING:  # pragma: no cover
+    from aimeval.client import AIMEval, AsyncAIMEval
+
+
+class _Replay:
+    """Beta wrapper around :mod:`aimeval._trace_io`.
+
+    Stays opt-in via ``client.beta.replay.save_trace(...)`` for the
+    same reason every premium SDK quarantines unstable surface:
+    discoverable via autocomplete, but the rename / removal is on
+    a separate timeline from the stable client.
+    """
+
+    def __init__(self, _client: "AIMEval | AsyncAIMEval"):
+        # Client kept on the instance only so a future beta feature
+        # that *does* hit the network has something to call. The replay
+        # helpers themselves are pure offline.
+        self._client = _client
+
+    save_trace = staticmethod(_trace_io.save_trace)
+    load_trace = staticmethod(_trace_io.load_trace)
+    load_trace_iter = staticmethod(_trace_io.load_trace_iter)
+    replay_trace = staticmethod(_trace_io.replay_trace)
+
+
+class Beta:
+    """Namespace for experimental features. See module docstring."""
+
+    def __init__(self, client: "AIMEval | AsyncAIMEval"):
+        self.replay = _Replay(client)
+
+
+__all__ = ["Beta"]

aimeval/_eval.py

@@ -0,0 +1,316 @@
+"""``aimeval.Eval`` + ``assert_test`` — the CI evaluation harness.
+
+Competitors expose a one-call eval harness (Braintrust ``Eval(name,
+data, task, scores)``, DeepEval ``evaluate(...)`` + ``assert_test``,
+LangSmith ``client.evaluate(...)``). This module is AIMEval's version,
+scoped honestly to how AIMEval actually scores.
+
+**How AIMEval scoring works (and why this layer is honest).** The
+metrics (visual faithfulness, hallucination, …) are computed
+server-side — the VLM judge + computation pipeline need the image,
+which the SDK doesn't have. So :func:`Eval` does NOT re-score locally.
+It:
+
+  1. launches a server-side run (``client.runs.evaluate``) — or takes an
+     already-finished run via :func:`eval_from_run`;
+  2. reads the run's server-computed per-metric scores;
+  3. interprets each score against the local :class:`~aimeval.metrics.Metric`
+     threshold you passed (direction-aware: lower-is-better metrics use
+     ``<=``);
+  4. returns a structured :class:`EvalResult` with a single ``passed``
+     boolean for CI.
+
+:func:`assert_test` is the pytest-friendly wrapper: it raises
+``AssertionError`` listing exactly which metrics fell below threshold —
+the DeepEval ``assert_test`` ergonomics, on AIMEval's server-scored
+runs.
+
+    import aimeval
+    from aimeval.metrics import VisualFaithfulness, Hallucination
+
+    result = aimeval.Eval(
+        name="ci-nightly", model=M, dataset=D, metrics=C,
+        scorers=[VisualFaithfulness(threshold=0.8), Hallucination(threshold=0.1)],
+    )
+    aimeval.assert_test(result)        # raises if any scorer failed
+"""
+from __future__ import annotations
+
+import os
+from typing import TYPE_CHECKING, Any, Sequence
+
+from aimeval._types import Resource, Run
+from aimeval.metrics import Metric, resolve
+
+if TYPE_CHECKING:  # pragma: no cover
+    from aimeval.client import AIMEval
+
+
+class EvalScore(Resource):
+    """One metric's outcome inside an :class:`EvalResult`."""
+
+    metric: str = ""
+    label: str = ""
+    score: float | None = None
+    threshold: float | None = None
+    operator: str = ">="
+    passed: bool = True
+
+    def __repr__(self) -> str:
+        mark = "✓" if self.passed else "✗"
+        thr = f" {self.operator} {self.threshold}" if self.threshold is not None else ""
+        return f"EvalScore({mark} {self.metric}={self.score}{thr})"
+
+
+class EvalResult(Resource):
+    """Outcome of an :func:`Eval` — server-scored run + threshold verdicts.
+
+    ``passed`` is the single CI signal: every scorer met its threshold
+    **and** the run's gate didn't fail.
+    """
+
+    name: str = ""
+    run_id: str = ""
+    gate_status: str | None = None
+    overall_score: float | None = None
+    scores: list[EvalScore] | None = None
+
+    @property
+    def passed(self) -> bool:
+        gate_ok = (self.gate_status or "").lower() != "fail"
+        scorers_ok = all(s.passed for s in (self.scores or []))
+        return gate_ok and scorers_ok
+
+    @property
+    def failures(self) -> list[EvalScore]:
+        return [s for s in (self.scores or []) if not s.passed]
+
+    def __repr__(self) -> str:
+        mark = "PASS" if self.passed else "FAIL"
+        return (
+            f"EvalResult({mark} name={self.name!r} run={self.run_id!r} "
+            f"scores={len(self.scores or [])} failures={len(self.failures)})"
+        )
+
+    def _repr_html_(self) -> str:
+        """Jupyter renders this HTML in a notebook cell."""
+        from aimeval._repr_html import eval_result_html
+        return eval_result_html(self)
+
+
+# ── score extraction ────────────────────────────────────────────────────
+
+def _all_scores(run: Run) -> dict[str, float]:
+    """Merge a run's judge + gt + top-level score maps into one slug→value
+    dict. Server returns scores split across ``judge_scores`` /
+    ``gt_scores``; we flatten so a metric lookup finds its value wherever
+    the pipeline emitted it."""
+    merged: dict[str, float] = {}
+    for source in (run.gt_scores, run.judge_scores):
+        if isinstance(source, dict):
+            for k, v in source.items():
+                if isinstance(v, (int, float)):
+                    merged[str(k)] = float(v)
+    return merged
+
+
+def _score_for(metric: Metric, scores: dict[str, float]) -> float | None:
+    """Find a metric's score, tolerating alias/label keys the server may
+    have used (e.g. ``reference_attribute_match`` for
+    ``reference_attr_match``)."""
+    if metric.slug in scores:
+        return scores[metric.slug]
+    # Try resolving each returned key back to the canonical slug.
+    for key, value in scores.items():
+        cls = resolve(key)
+        if cls is not None and cls.slug == metric.slug:
+            return value
+    return None
+
+
+def _build_scores(
+    run: Run, scorers: Sequence[Metric] | None,
+) -> list[EvalScore]:
+    raw = _all_scores(run)
+    out: list[EvalScore] = []
+    if scorers:
+        for m in scorers:
+            value = _score_for(m, raw)
+            out.append(EvalScore(
+                metric=m.slug, label=m.label, score=value,
+                threshold=m.threshold, operator=m.operator,
+                passed=m.passed(value),
+            ))
+    else:
+        # No explicit scorers → report every returned metric, no threshold.
+        for slug, value in raw.items():
+            cls = resolve(slug)
+            out.append(EvalScore(
+                metric=(cls.slug if cls else slug),
+                label=(cls.label if cls else slug),
+                score=value, threshold=None,
+                operator=">=", passed=True,
+            ))
+    return out
+
+
+def _coerce_scorers(
+    scorers: Sequence[Metric | str] | None,
+) -> list[Metric] | None:
+    if scorers is None:
+        return None
+    out: list[Metric] = []
+    for s in scorers:
+        if isinstance(s, Metric):
+            out.append(s)
+        elif isinstance(s, str):
+            cls = resolve(s)
+            if cls is None:
+                raise KeyError(f"unknown metric {s!r}")
+            out.append(cls())
+        else:
+            raise TypeError(f"scorer must be a Metric or slug str, got {type(s)}")
+    return out
+
+
+def _resolve_client(client: "AIMEval | None") -> "AIMEval":
+    if client is not None:
+        return client
+    # Lazy import to avoid a hard import cycle at module load.
+    from aimeval.client import AIMEval
+    # Honest failure: if env isn't set, AIMEval() raises a clear error —
+    # we don't silently fabricate a client.
+    if not os.environ.get("AIMEVAL_API_KEY") or not os.environ.get("AIMEVAL_BASE_URL"):
+        raise RuntimeError(
+            "Eval() needs a client: pass client=AIMEval(...) or set "
+            "AIMEVAL_API_KEY + AIMEVAL_BASE_URL env vars.",
+        )
+    return AIMEval()
+
+
+# ── public API ──────────────────────────────────────────────────────────
+
+def Eval(
+    name: str,
+    *,
+    model: str,
+    dataset: str,
+    metrics: str,
+    scorers: Sequence[Metric | str] | None = None,
+    client: "AIMEval | None" = None,
+    timeout: int = 600,
+    progress: bool | str = "auto",
+    **run_kwargs: Any,
+) -> EvalResult:
+    """Run a server-side evaluation and interpret it against thresholds.
+
+    Args:
+        name: Run name.
+        model / dataset / metrics: IDs forwarded to ``runs.evaluate``
+            (``metrics`` is the metric-collection ID that selects which
+            metrics the server computes).
+        scorers: List of :class:`~aimeval.metrics.Metric` instances (with
+            thresholds) or metric slugs. Each is checked against the run's
+            server-computed score, direction-aware. Omit to report all
+            returned scores without thresholds.
+        client: An ``AIMEval``; defaults to one built from env vars.
+        timeout / progress / run_kwargs: Forwarded to ``runs.evaluate``.
+
+    Returns:
+        :class:`EvalResult` — ``.passed`` is the CI signal.
+    """
+    cl = _resolve_client(client)
+    scorer_objs = _coerce_scorers(scorers)
+    run = cl.runs.evaluate(
+        name=name, model=model, dataset=dataset, metrics=metrics,
+        timeout=timeout, progress=progress, **run_kwargs,
+    )
+    return eval_from_run(run, scorers=scorer_objs, name=name)
+
+
+def eval_from_run(
+    run: Run,
+    *,
+    scorers: Sequence[Metric | str] | None = None,
+    name: str | None = None,
+) -> EvalResult:
+    """Interpret an already-finished :class:`Run` against thresholds.
+
+    Useful when you ran the evaluation elsewhere (UI / a prior CI step)
+    and only want the threshold verdict::
+
+        run = client.runs.retrieve(run_id)
+        result = aimeval.eval_from_run(run, scorers=[Hallucination(threshold=0.1)])
+    """
+    scorer_objs = _coerce_scorers(scorers)
+    return EvalResult(
+        name=name or run.name or "",
+        run_id=run.id,
+        gate_status=run.gate_status,
+        overall_score=run.overall_score,
+        scores=_build_scores(run, scorer_objs),
+    )
+
+
+class EvalAssertionError(AssertionError):
+    """Raised by :func:`assert_test` when a scorer falls below threshold."""
+
+
+def assert_test(
+    result: EvalResult | Run,
+    scorers: Sequence[Metric | str] | None = None,
+) -> EvalResult:
+    """Assert that every scorer met its threshold; raise otherwise.
+
+    pytest-friendly (DeepEval ``assert_test`` ergonomics)::
+
+        def test_nightly_quality():
+            result = aimeval.Eval(name="t", model=M, dataset=D, metrics=C,
+                                  scorers=[VisualFaithfulness(threshold=0.8)])
+            aimeval.assert_test(result)
+
+    Accepts an :class:`EvalResult` directly, or a :class:`Run` plus
+    ``scorers`` (interpreted on the spot). Returns the result on success
+    so it can be inspected.
+    """
+    if isinstance(result, Run):
+        result = eval_from_run(result, scorers=scorers)
+    elif scorers is not None and not (result.scores or []):
+        # An EvalResult built without scorers but assert_test was given
+        # some — re-interpret would need the raw run; we can only check
+        # what's present. Fail loudly rather than silently ignore.
+        raise ValueError(
+            "assert_test received scorers but the EvalResult has no scores; "
+            "pass scorers to Eval()/eval_from_run() instead.",
+        )
+
+    failures = result.failures
+    if failures:
+        lines = "\n".join(
+            f"  ✗ {f.label} ({f.metric}): score={f.score} "
+            f"{f.operator} {f.threshold} → FAILED"
+            for f in failures
+        )
+        gate = (result.gate_status or "").lower()
+        gate_line = (
+            f"\n  ✗ gate_status={result.gate_status}" if gate == "fail" else ""
+        )
+        raise EvalAssertionError(
+            f"Eval '{result.name}' (run {result.run_id}) failed "
+            f"{len(failures)} metric threshold(s):\n{lines}{gate_line}",
+        )
+    if (result.gate_status or "").lower() == "fail":
+        raise EvalAssertionError(
+            f"Eval '{result.name}' (run {result.run_id}) — gate_status=fail",
+        )
+    return result
+
+
+__all__ = [
+    "Eval",
+    "eval_from_run",
+    "assert_test",
+    "EvalResult",
+    "EvalScore",
+    "EvalAssertionError",
+]

aimeval/_exceptions.py

@@ -0,0 +1,119 @@
+"""Error hierarchy.
+
+Catch-by-status pattern (industry standard — OpenAI, Anthropic, Stripe)::
+
+    try:
+        client.runs.create(...)
+    except aimeval.RateLimitError:
+        time.sleep(30)
+    except aimeval.AuthenticationError:
+        sys.exit("bad key")
+    except aimeval.AIMEvalError:
+        ...  # everything else
+"""
+from __future__ import annotations
+
+
+class AIMEvalError(Exception):
+    """Base for every error the SDK raises.
+
+    Industry-standard error context (OpenAI / Anthropic / Stripe):
+    every error carries the upstream ``X-Request-Id`` so users can copy
+    it into a support ticket. Backend already attaches one via
+    ``config.api.request_id`` middleware.
+    """
+
+    def __init__(
+        self,
+        message: str,
+        *,
+        status_code: int = 0,
+        details: dict | None = None,
+        request_id: str | None = None,
+        response: object | None = None,
+    ):
+        super().__init__(message)
+        self.status_code = status_code
+        self.details = details or {}
+        self.request_id = request_id
+        # Raw httpx.Response for power users that need headers / body.
+        # Kept as ``object`` to avoid leaking httpx into the public type.
+        self.response = response
+
+    def __str__(self) -> str:
+        base = super().__str__()
+        if self.request_id:
+            return f"{base} (request_id={self.request_id})"
+        return base
+
+
+class APIConnectionError(AIMEvalError):
+    """Network / DNS / TLS failure — no HTTP response was received."""
+
+
+class APITimeoutError(APIConnectionError):
+    """Request didn't finish within the configured timeout."""
+
+
+class AuthenticationError(AIMEvalError):
+    """401 — API key missing, invalid, or revoked."""
+
+
+class PermissionError_(AIMEvalError):  # avoid shadowing builtin
+    """403 — key valid but not allowed to do this."""
+
+
+class NotFoundError(AIMEvalError):
+    """404 — resource doesn't exist (or isn't visible to this key)."""
+
+
+class ConflictError(AIMEvalError):
+    """409 — concurrent edit or duplicate idempotency key."""
+
+
+class BadRequestError(AIMEvalError):
+    """400 / 422 — request shape is wrong; ``details`` has field-level info."""
+
+
+class RateLimitError(AIMEvalError):
+    """429 — too many requests; retry after ``Retry-After`` seconds."""
+
+
+class APIServerError(AIMEvalError):
+    """5xx — server-side failure."""
+
+
+class AIMEvalSecurityWarning(UserWarning):
+    """Warned when SDK usage risks leaking credentials.
+
+    Filtered with ``warnings.simplefilter('ignore', AIMEvalSecurityWarning)``.
+    """
+
+
+def from_status(
+    status_code: int,
+    message: str,
+    details: dict | None = None,
+    *,
+    request_id: str | None = None,
+    response: object | None = None,
+) -> AIMEvalError:
+    """Map an HTTP status code to the most specific subclass."""
+    cls = _STATUS_MAP.get(status_code)
+    if cls is None:
+        cls = APIServerError if status_code >= 500 else AIMEvalError
+    return cls(
+        message, status_code=status_code, details=details,
+        request_id=request_id, response=response,
+    )
+
+
+_STATUS_MAP: dict[int, type[AIMEvalError]] = {
+    400: BadRequestError,
+    401: AuthenticationError,
+    403: PermissionError_,
+    404: NotFoundError,
+    409: ConflictError,
+    422: BadRequestError,
+    429: RateLimitError,
+}