aeri-python 4.0.0__py3-none-any.whl

aeri/__init__.py

@@ -0,0 +1,72 @@
+""".. include:: ../README.md"""
+
+from aeri.batch_evaluation import (
+    BatchEvaluationResult,
+    BatchEvaluationResumeToken,
+    CompositeEvaluatorFunction,
+    EvaluatorInputs,
+    EvaluatorStats,
+    MapperFunction,
+)
+from aeri.experiment import Evaluation
+
+from ._client import client as _client_module
+from ._client.attributes import AeriOtelSpanAttributes
+from ._client.constants import ObservationTypeLiteral
+from ._client.get_client import get_client
+from ._client.observe import observe
+from ._client.propagation import propagate_attributes
+from ._client.span import (
+    AeriAgent,
+    AeriChain,
+    AeriEmbedding,
+    AeriEvaluator,
+    AeriEvent,
+    AeriGeneration,
+    AeriGuardrail,
+    AeriRetriever,
+    AeriSpan,
+    AeriTool,
+)
+from .span_filter import (
+    KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES,
+    is_default_export_span,
+    is_genai_span,
+    is_known_llm_instrumentor,
+    is_aeri_span,
+)
+
+Aeri = _client_module.Aeri
+
+__all__ = [
+    "Aeri",
+    "get_client",
+    "observe",
+    "propagate_attributes",
+    "ObservationTypeLiteral",
+    "AeriSpan",
+    "AeriGeneration",
+    "AeriEvent",
+    "AeriOtelSpanAttributes",
+    "AeriAgent",
+    "AeriTool",
+    "AeriChain",
+    "AeriEmbedding",
+    "AeriEvaluator",
+    "AeriRetriever",
+    "AeriGuardrail",
+    "Evaluation",
+    "EvaluatorInputs",
+    "MapperFunction",
+    "CompositeEvaluatorFunction",
+    "EvaluatorStats",
+    "BatchEvaluationResumeToken",
+    "BatchEvaluationResult",
+    "is_default_export_span",
+    "is_aeri_span",
+    "is_genai_span",
+    "is_known_llm_instrumentor",
+    "KNOWN_LLM_INSTRUMENTATION_SCOPE_PREFIXES",
+    "experiment",
+    "api",
+]

aeri/_client/_validation.py

@@ -0,0 +1,204 @@
+"""Pydantic V2 strict validation models for all public Aeri client entry-points.
+
+Every payload that a caller passes to ``Aeri.start_observation``,
+``Aeri.create_score``, or any other public method is validated through one of
+these models *before* being passed deeper into the SDK.
+
+Design choices
+──────────────
+• ``model_config = ConfigDict(strict=True)`` — rejects silent coercions (e.g.
+  "1" → 1) that can hide bugs in caller code.
+• ``extra="forbid"`` — fails fast on unknown/misspelled fields.
+• All fields that map to API concepts reuse the auto-generated Fern types (e.g.
+  ``MapValue``, ``ScoreBody``) so there's a single source of truth.
+• Validators use Annotated + Field constraints rather than ``@validator`` /
+  ``@root_validator`` (Pydantic V1 patterns) for cleaner V2 semantics.
+"""
+
+from __future__ import annotations
+
+from datetime import datetime
+from typing import Annotated, Any, Dict, Literal, Optional, Union
+
+from pydantic import BaseModel, ConfigDict, Field, field_validator, model_validator
+
+from aeri._client.constants import (
+    ObservationTypeGenerationLike,
+    ObservationTypeLiteralNoEvent,
+    get_observation_types_list,
+)
+from aeri.api import MapValue, ScoreBody
+from aeri.types import ScoreDataType, SpanLevel
+
+# ─── Shared constraints ───────────────────────────────────────────────────────
+
+_NonEmptyStr = Annotated[str, Field(min_length=1)]
+
+
+# ─── Observation (span / generation / agent / tool …) ────────────────────────
+
+class _GenerationFields(BaseModel):
+    """Extra fields that only apply to generation-like observations."""
+
+    model_config = ConfigDict(strict=True, extra="forbid")
+
+    completion_start_time: Optional[datetime] = None
+    model: Optional[str] = None
+    model_parameters: Optional[Dict[str, MapValue]] = None
+    usage_details: Optional[Dict[str, int]] = None
+    cost_details: Optional[Dict[str, float]] = None
+    prompt: Optional[Any] = None  # PromptClient — avoid circular import
+
+
+class ObservationInput(BaseModel):
+    """Validated payload for ``Aeri.start_observation`` /
+    ``Aeri.start_as_current_observation``."""
+
+    model_config = ConfigDict(strict=True, extra="forbid", populate_by_name=True)
+
+    name: _NonEmptyStr
+    as_type: ObservationTypeLiteralNoEvent = "span"
+    input: Optional[Any] = None
+    output: Optional[Any] = None
+    metadata: Optional[Any] = None
+    version: Optional[str] = None
+    level: Optional[SpanLevel] = None
+    status_message: Optional[str] = None
+
+    # Generation-only fields — allowed on any type but only forwarded when
+    # as_type is in ObservationTypeGenerationLike
+    completion_start_time: Optional[datetime] = None
+    model: Optional[str] = None
+    model_parameters: Optional[Dict[str, MapValue]] = None
+    usage_details: Optional[Dict[str, int]] = None
+    cost_details: Optional[Dict[str, float]] = None
+    prompt: Optional[Any] = None
+
+    @model_validator(mode="after")
+    def _warn_generation_fields_on_non_generation(self) -> "ObservationInput":
+        generation_types = get_observation_types_list(ObservationTypeGenerationLike)
+        if self.as_type not in generation_types:
+            gen_only = [
+                f
+                for f in ("completion_start_time", "model", "model_parameters",
+                           "usage_details", "cost_details", "prompt")
+                if getattr(self, f) is not None
+            ]
+            if gen_only:
+                import warnings
+                warnings.warn(
+                    f"Fields {gen_only} are only meaningful for generation-like "
+                    f"observations (as_type in {generation_types}), but "
+                    f"as_type={self.as_type!r} was given. They will be ignored.",
+                    stacklevel=4,
+                )
+        return self
+
+
+# ─── Score creation ───────────────────────────────────────────────────────────
+
+class ScoreInput(BaseModel):
+    """Validated payload for ``Aeri.create_score`` and the span ``.score()`` helpers."""
+
+    model_config = ConfigDict(strict=True, extra="forbid")
+
+    name: _NonEmptyStr
+    value: Union[float, int, bool, str]
+    data_type: Optional[ScoreDataType] = None
+    comment: Optional[str] = None
+    config_id: Optional[str] = None
+    trace_id: Optional[str] = None
+    observation_id: Optional[str] = None
+
+    @field_validator("value")
+    @classmethod
+    def _coerce_bool_to_int(cls, v: Any) -> Any:
+        # Preserve bool as-is; API handles BOOLEAN data_type separately
+        return v
+
+    @model_validator(mode="after")
+    def _validate_data_type_matches_value(self) -> "ScoreInput":
+        if self.data_type is None:
+            return self
+        # Numeric
+        if self.data_type == "NUMERIC" and not isinstance(self.value, (int, float)):
+            value_repr = repr(self.value)
+            msg = f"data_type='NUMERIC' requires a numeric value; got {value_repr!r}"
+            raise ValueError(msg)
+        # Boolean
+        if self.data_type == "BOOLEAN" and not isinstance(self.value, (bool, int)):
+            value_repr = repr(self.value)
+            msg = f"data_type='BOOLEAN' requires a bool/int value (0 or 1); got {value_repr!r}"
+            raise ValueError(msg)
+        # Categorical
+        if self.data_type == "CATEGORICAL" and not isinstance(self.value, str):
+            value_repr = repr(self.value)
+            msg = f"data_type='CATEGORICAL' requires a string value; got {value_repr!r}"
+            raise ValueError(msg)
+        return self
+
+
+# ─── Client constructor config ────────────────────────────────────────────────
+
+class AeriClientConfig(BaseModel):
+    """Validates all constructor parameters for the ``Aeri`` client.
+
+    This is applied in ``Aeri.__init__`` after environment variable resolution
+    so that misconfigured callers get a clear error immediately.
+    """
+
+    model_config = ConfigDict(strict=False, extra="forbid")  # lenient: coerce env strs
+
+    public_key: Optional[str] = None
+    secret_key: Optional[str] = None
+    base_url: str = "https://api.aeri.com"
+    timeout: int = Field(default=5, ge=1, le=300)
+    debug: bool = False
+    tracing_enabled: bool = True
+    flush_at: int = Field(default=15, ge=1, le=10_000)
+    flush_interval: float = Field(default=1.0, ge=0.1, le=3600.0)
+    environment: Optional[str] = None
+    release: Optional[str] = None
+    media_upload_thread_count: int = Field(default=1, ge=1, le=32)
+    sample_rate: float = Field(default=1.0, ge=0.0, le=1.0)
+
+    @field_validator("environment")
+    @classmethod
+    def _validate_environment_name(cls, v: Optional[str]) -> Optional[str]:
+        if v is None:
+            return v
+        if v.lower().startswith("aeri"):
+            msg = "environment name must not start with 'aeri'"
+            raise ValueError(msg)
+        import re
+        if not re.fullmatch(r"[a-z0-9_\-]+", v):
+            msg = (
+                f"environment must be lowercase alphanumeric with hyphens/underscores; "
+                f"got {v!r}"
+            )
+            raise ValueError(msg)
+        return v
+
+
+# ─── Trace context ────────────────────────────────────────────────────────────
+
+class TraceContextInput(BaseModel):
+    """Validates the ``trace_context`` dict passed to ``start_observation``."""
+
+    model_config = ConfigDict(strict=True, extra="forbid")
+
+    trace_id: Optional[str] = None
+    parent_span_id: Optional[str] = None
+
+    @field_validator("trace_id", "parent_span_id")
+    @classmethod
+    def _must_be_hex(cls, v: Optional[str], info: Any) -> Optional[str]:
+        if v is None:
+            return v
+        try:
+            int(v, 16)
+        except ValueError:
+            field_repr = repr(info.field_name)
+            msg = f"{field_repr} must be a hexadecimal string; got {v!r}"
+            raise ValueError(msg)
+        return v

aeri/_client/attributes.py

@@ -0,0 +1,188 @@
+"""Span attribute management for Aeri OpenTelemetry integration.
+
+This module defines constants and functions for managing OpenTelemetry span attributes
+used by Aeri. It provides a structured approach to creating and manipulating
+attributes for different span types (trace, span, generation) while ensuring consistency.
+
+The module includes:
+- Attribute name constants organized by category
+- Functions to create attribute dictionaries for different entity types
+- Utilities for serializing and processing attribute values
+"""
+
+import json
+from datetime import datetime
+from typing import Any, Dict, Literal, Optional, Union
+
+from aeri._client.constants import (
+    ObservationTypeGenerationLike,
+    ObservationTypeSpanLike,
+)
+from aeri._utils.serializer import EventSerializer
+from aeri.api import MapValue
+from aeri.model import PromptClient
+from aeri.types import SpanLevel
+
+
+class AeriOtelSpanAttributes:
+    # Aeri-Trace attributes
+    TRACE_NAME = "aeri.trace.name"
+    TRACE_USER_ID = "user.id"
+    TRACE_SESSION_ID = "session.id"
+    TRACE_TAGS = "aeri.trace.tags"
+    TRACE_PUBLIC = "aeri.trace.public"
+    TRACE_METADATA = "aeri.trace.metadata"
+    TRACE_INPUT = "aeri.trace.input"
+    TRACE_OUTPUT = "aeri.trace.output"
+
+    # Aeri-observation attributes
+    OBSERVATION_TYPE = "aeri.observation.type"
+    OBSERVATION_METADATA = "aeri.observation.metadata"
+    OBSERVATION_LEVEL = "aeri.observation.level"
+    OBSERVATION_STATUS_MESSAGE = "aeri.observation.status_message"
+    OBSERVATION_INPUT = "aeri.observation.input"
+    OBSERVATION_OUTPUT = "aeri.observation.output"
+
+    # Aeri-observation of type Generation attributes
+    OBSERVATION_COMPLETION_START_TIME = "aeri.observation.completion_start_time"
+    OBSERVATION_MODEL = "aeri.observation.model.name"
+    OBSERVATION_MODEL_PARAMETERS = "aeri.observation.model.parameters"
+    OBSERVATION_USAGE_DETAILS = "aeri.observation.usage_details"
+    OBSERVATION_COST_DETAILS = "aeri.observation.cost_details"
+    OBSERVATION_PROMPT_NAME = "aeri.observation.prompt.name"
+    OBSERVATION_PROMPT_VERSION = "aeri.observation.prompt.version"
+
+    # General
+    ENVIRONMENT = "aeri.environment"
+    RELEASE = "aeri.release"
+    VERSION = "aeri.version"
+
+    # Internal
+    AS_ROOT = "aeri.internal.as_root"
+
+    # Experiments
+    EXPERIMENT_ID = "aeri.experiment.id"
+    EXPERIMENT_NAME = "aeri.experiment.name"
+    EXPERIMENT_DESCRIPTION = "aeri.experiment.description"
+    EXPERIMENT_METADATA = "aeri.experiment.metadata"
+    EXPERIMENT_DATASET_ID = "aeri.experiment.dataset.id"
+    EXPERIMENT_ITEM_ID = "aeri.experiment.item.id"
+    EXPERIMENT_ITEM_EXPECTED_OUTPUT = "aeri.experiment.item.expected_output"
+    EXPERIMENT_ITEM_METADATA = "aeri.experiment.item.metadata"
+    EXPERIMENT_ITEM_ROOT_OBSERVATION_ID = "aeri.experiment.item.root_observation_id"
+
+
+def create_trace_attributes(
+    *,
+    input: Optional[Any] = None,
+    output: Optional[Any] = None,
+    public: Optional[bool] = None,
+) -> dict:
+    attributes = {
+        AeriOtelSpanAttributes.TRACE_INPUT: _serialize(input),
+        AeriOtelSpanAttributes.TRACE_OUTPUT: _serialize(output),
+        AeriOtelSpanAttributes.TRACE_PUBLIC: public,
+    }
+
+    return {k: v for k, v in attributes.items() if v is not None}
+
+
+def create_span_attributes(
+    *,
+    metadata: Optional[Any] = None,
+    input: Optional[Any] = None,
+    output: Optional[Any] = None,
+    level: Optional[SpanLevel] = None,
+    status_message: Optional[str] = None,
+    version: Optional[str] = None,
+    observation_type: Optional[
+        Union[ObservationTypeSpanLike, Literal["event"]]
+    ] = "span",
+) -> dict:
+    attributes = {
+        AeriOtelSpanAttributes.OBSERVATION_TYPE: observation_type,
+        AeriOtelSpanAttributes.OBSERVATION_LEVEL: level,
+        AeriOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE: status_message,
+        AeriOtelSpanAttributes.VERSION: version,
+        AeriOtelSpanAttributes.OBSERVATION_INPUT: _serialize(input),
+        AeriOtelSpanAttributes.OBSERVATION_OUTPUT: _serialize(output),
+        **_flatten_and_serialize_metadata(metadata, "observation"),
+    }
+
+    return {k: v for k, v in attributes.items() if v is not None}
+
+
+def create_generation_attributes(
+    *,
+    name: Optional[str] = None,
+    completion_start_time: Optional[datetime] = None,
+    metadata: Optional[Any] = None,
+    level: Optional[SpanLevel] = None,
+    status_message: Optional[str] = None,
+    version: Optional[str] = None,
+    model: Optional[str] = None,
+    model_parameters: Optional[Dict[str, MapValue]] = None,
+    input: Optional[Any] = None,
+    output: Optional[Any] = None,
+    usage_details: Optional[Dict[str, int]] = None,
+    cost_details: Optional[Dict[str, float]] = None,
+    prompt: Optional[PromptClient] = None,
+    observation_type: Optional[ObservationTypeGenerationLike] = "generation",
+) -> dict:
+    attributes = {
+        AeriOtelSpanAttributes.OBSERVATION_TYPE: observation_type,
+        AeriOtelSpanAttributes.OBSERVATION_LEVEL: level,
+        AeriOtelSpanAttributes.OBSERVATION_STATUS_MESSAGE: status_message,
+        AeriOtelSpanAttributes.VERSION: version,
+        AeriOtelSpanAttributes.OBSERVATION_INPUT: _serialize(input),
+        AeriOtelSpanAttributes.OBSERVATION_OUTPUT: _serialize(output),
+        AeriOtelSpanAttributes.OBSERVATION_MODEL: model,
+        AeriOtelSpanAttributes.OBSERVATION_PROMPT_NAME: prompt.name
+        if prompt and not prompt.is_fallback
+        else None,
+        AeriOtelSpanAttributes.OBSERVATION_PROMPT_VERSION: prompt.version
+        if prompt and not prompt.is_fallback
+        else None,
+        AeriOtelSpanAttributes.OBSERVATION_USAGE_DETAILS: _serialize(usage_details),
+        AeriOtelSpanAttributes.OBSERVATION_COST_DETAILS: _serialize(cost_details),
+        AeriOtelSpanAttributes.OBSERVATION_COMPLETION_START_TIME: _serialize(
+            completion_start_time
+        ),
+        AeriOtelSpanAttributes.OBSERVATION_MODEL_PARAMETERS: _serialize(
+            model_parameters
+        ),
+        **_flatten_and_serialize_metadata(metadata, "observation"),
+    }
+
+    return {k: v for k, v in attributes.items() if v is not None}
+
+
+def _serialize(obj: Any) -> Optional[str]:
+    if obj is None or isinstance(obj, str):
+        return obj
+
+    return json.dumps(obj, cls=EventSerializer)
+
+
+def _flatten_and_serialize_metadata(
+    metadata: Any, type: Literal["observation", "trace"]
+) -> dict:
+    prefix = (
+        AeriOtelSpanAttributes.OBSERVATION_METADATA
+        if type == "observation"
+        else AeriOtelSpanAttributes.TRACE_METADATA
+    )
+
+    metadata_attributes: Dict[str, Union[str, int, None]] = {}
+
+    if not isinstance(metadata, dict):
+        metadata_attributes[prefix] = _serialize(metadata)
+    else:
+        for key, value in metadata.items():
+            metadata_attributes[f"{prefix}.{key}"] = (
+                value
+                if isinstance(value, str) or isinstance(value, int)
+                else _serialize(value)
+            )
+
+    return metadata_attributes