agent-reliability 1.0.0__py3-none-any.whl

agent_reliability/__init__.py

@@ -0,0 +1,14 @@
+"""Agent Reliability SDK.
+
+Vendor-neutral reliability primitives for operating autonomous AI agents.
+
+See docs/GA_CONTRACT.md and docs/COMPATIBILITY.md for the stability policy.
+Public APIs live in focused subpackages; the root package intentionally
+exposes only the distribution version.
+"""
+
+from __future__ import annotations
+
+__version__ = "1.0.0"
+
+__all__ = ["__version__"]

agent_reliability/adapters/__init__.py

@@ -0,0 +1,31 @@
+"""Adapters: concrete implementations of ports.
+
+The default clock, id generator, and event sinks arrived in M2. M3's optional
+OpenTelemetry adapter lives in ``agent_reliability.adapters.otel`` and is not
+imported here, preserving base-install import safety.
+
+This is the only layer permitted to depend on a specific vendor SDK,
+transport, or agent framework. Adapters implement ports; they are never
+imported by ``domain`` or ``application``.
+
+The exports of this subpackage are part of the stable 1.0 contract documented
+in docs/GA_CONTRACT.md.
+"""
+
+from __future__ import annotations
+
+from agent_reliability.adapters.event_sinks import (
+    CompositeEventSink,
+    InMemoryEventSink,
+    NoOpEventSink,
+)
+from agent_reliability.adapters.system_clock import SystemClock
+from agent_reliability.adapters.uuid_run_id_generator import UuidRunIdGenerator
+
+__all__ = [
+    "CompositeEventSink",
+    "InMemoryEventSink",
+    "NoOpEventSink",
+    "SystemClock",
+    "UuidRunIdGenerator",
+]

agent_reliability/adapters/event_sinks.py

@@ -0,0 +1,82 @@
+"""Default ``EventSink`` implementations.
+
+``NoOpEventSink`` is what ``AgentReliability()`` uses when no sink is
+given — a library should not silently print to the console by default,
+and doing nothing observable until a caller opts in is the least
+surprising default for production embedding (docs/SDK_DESIGN.md).
+"""
+
+from __future__ import annotations
+
+import threading
+
+from agent_reliability.ports.event_sink import EventSink
+from agent_reliability.ports.events import InstrumentationEvent
+
+__all__ = ["CompositeEventSink", "InMemoryEventSink", "NoOpEventSink"]
+
+
+class NoOpEventSink:
+    """Discards every event. The default sink."""
+
+    def emit(self, event: InstrumentationEvent) -> None:
+        return None
+
+
+class InMemoryEventSink:
+    """Appends every event to a list, held in memory for the sink's
+    lifetime.
+
+    Intended for tests and local examples — **not** a production
+    default. Unbounded accumulation of every event for the life of the
+    process is exactly the kind of memory growth this project's
+    engineering principles warn against as a default (docs/SDK_DESIGN.md,
+    "Memory safety"). ``events`` is safe to read/append from multiple
+    threads (protected by an internal lock); it is still an unbounded,
+    ever-growing list, by design, for this sink.
+    """
+
+    def __init__(self) -> None:
+        self._lock = threading.Lock()
+        self._events: list[InstrumentationEvent] = []
+
+    def emit(self, event: InstrumentationEvent) -> None:
+        with self._lock:
+            self._events.append(event)
+
+    @property
+    def events(self) -> list[InstrumentationEvent]:
+        """A snapshot copy of the events received so far."""
+        with self._lock:
+            return list(self._events)
+
+    def clear(self) -> None:
+        with self._lock:
+            self._events.clear()
+
+
+class CompositeEventSink:
+    """Fans one event out to multiple sinks.
+
+    Each child's ``emit`` is called independently; one child raising
+    does not stop delivery to the others (the exception is allowed to
+    propagate to the SDK's own failure-isolation wrapper around this
+    composite's ``emit`` call, which reports it via diagnostics exactly
+    as it would for a single misbehaving sink — see
+    docs/adr/0004-instrumentation-failure-isolation.md). The *first*
+    child to raise is what gets reported; later children still run.
+    """
+
+    def __init__(self, sinks: list[EventSink]) -> None:
+        self._sinks = list(sinks)
+
+    def emit(self, event: InstrumentationEvent) -> None:
+        first_error: Exception | None = None
+        for sink in self._sinks:
+            try:
+                sink.emit(event)
+            except Exception as exc:
+                if first_error is None:
+                    first_error = exc
+        if first_error is not None:
+            raise first_error

agent_reliability/adapters/otel/__init__.py

@@ -0,0 +1,13 @@
+"""Optional OpenTelemetry interoperability adapter.
+
+Install ``agent-reliability[otel]`` before importing this module. The adapter
+uses the OpenTelemetry API but never configures a provider or exporter.
+"""
+
+from __future__ import annotations
+
+from agent_reliability.adapters.otel.run_context import (
+    OpenTelemetryRunContextBridge,
+)
+
+__all__ = ["OpenTelemetryRunContextBridge"]

agent_reliability/adapters/otel/run_context.py

@@ -0,0 +1,109 @@
+"""OpenTelemetry implementation of the run-context bridge port."""
+
+from __future__ import annotations
+
+from contextlib import AbstractContextManager, suppress
+
+from opentelemetry import trace
+from opentelemetry.trace import Span, SpanKind, Status, StatusCode, Tracer
+
+from agent_reliability import __version__
+from agent_reliability.domain import AgentRun, RunStatus
+from agent_reliability.ports.run_context import RunContextScope
+
+__all__ = ["OpenTelemetryRunContextBridge"]
+
+_INSTRUMENTATION_SCOPE_NAME = "agent_reliability"
+_SPAN_NAME = "invoke_agent"
+_SCHEMA_VERSION = "1"
+
+
+class OpenTelemetryRunContextBridge:
+    """Creates a current internal span for each initialized SDK run.
+
+    Passing a tracer is useful for explicit provider ownership and tests. If
+    omitted, the bridge obtains a tracer from the host-configured global
+    provider; it never installs or changes that provider.
+    """
+
+    def __init__(self, tracer: Tracer | None = None) -> None:
+        if tracer is not None and not isinstance(tracer, Tracer):
+            raise TypeError("tracer must implement opentelemetry.trace.Tracer")
+        self._tracer = (
+            tracer
+            if tracer is not None
+            else trace.get_tracer(_INSTRUMENTATION_SCOPE_NAME, __version__)
+        )
+
+    def start(self, run: AgentRun) -> RunContextScope:
+        attributes: dict[str, str] = {
+            "gen_ai.operation.name": "invoke_agent",
+            "gen_ai.agent.name": run.agent.name,
+            "agent_reliability.schema.version": _SCHEMA_VERSION,
+            "agent_reliability.agent.id": run.agent.agent_id,
+            "agent_reliability.agent.version": run.agent.version,
+            "agent_reliability.run.id": run.run_id,
+        }
+        if run.agent.environment is not None:
+            attributes["agent_reliability.agent.environment"] = run.agent.environment
+        if run.parent_run_id is not None:
+            attributes["agent_reliability.run.parent_id"] = run.parent_run_id
+
+        span = self._tracer.start_span(
+            _SPAN_NAME, kind=SpanKind.INTERNAL, attributes=attributes
+        )
+        activation = trace.use_span(
+            span,
+            end_on_exit=False,
+            record_exception=False,
+            set_status_on_exception=False,
+        )
+        try:
+            activation.__enter__()
+        except BaseException:
+            # Preserve the activation failure. There is no active scope for
+            # the SDK to finish, and the host runtime owns any additional
+            # OpenTelemetry diagnostics.
+            with suppress(BaseException):
+                span.end()
+            raise
+        return _OpenTelemetryRunContextScope(span=span, activation=activation)
+
+
+class _OpenTelemetryRunContextScope:
+    def __init__(self, *, span: Span, activation: AbstractContextManager[Span]) -> None:
+        self._span = span
+        self._activation = activation
+        self._closed = False
+
+    def finish(self, *, status: RunStatus, exception_type: str | None) -> None:
+        if self._closed:
+            return
+
+        first_error: BaseException | None = None
+        try:
+            try:
+                self._span.set_attribute("agent_reliability.run.status", status.value)
+                if status is RunStatus.FAILED:
+                    if exception_type is not None:
+                        self._span.set_attribute("error.type", exception_type)
+                    self._span.set_status(Status(StatusCode.ERROR))
+            except BaseException as exc:
+                first_error = exc
+
+            try:
+                self._activation.__exit__(None, None, None)
+            except BaseException as exc:
+                if first_error is None:
+                    first_error = exc
+
+            try:
+                self._span.end()
+            except BaseException as exc:
+                if first_error is None:
+                    first_error = exc
+        finally:
+            self._closed = True
+
+        if first_error is not None:
+            raise first_error

agent_reliability/adapters/system_clock.py

@@ -0,0 +1,14 @@
+"""The default ``Clock`` implementation."""
+
+from __future__ import annotations
+
+from datetime import UTC, datetime
+
+__all__ = ["SystemClock"]
+
+
+class SystemClock:
+    """Reads the real system clock, normalized to UTC."""
+
+    def now(self) -> datetime:
+        return datetime.now(UTC)

agent_reliability/adapters/uuid_run_id_generator.py

@@ -0,0 +1,15 @@
+"""The default ``RunIdGenerator`` implementation."""
+
+from __future__ import annotations
+
+import uuid
+
+__all__ = ["UuidRunIdGenerator"]
+
+
+class UuidRunIdGenerator:
+    """Generates run ids via ``uuid.uuid4()`` — random, not sequential,
+    globally-unique-compatible with no coordination required."""
+
+    def generate(self) -> str:
+        return str(uuid.uuid4())

agent_reliability/application/__init__.py

@@ -0,0 +1,3 @@
+"""Reserved internal application-layer namespace; exports no public API."""
+
+from __future__ import annotations

agent_reliability/domain/__init__.py

@@ -0,0 +1,66 @@
+"""Domain layer: pure reliability concepts (Run, Evaluation, SLI, SLO,
+Error Budget). See docs/DOMAIN_MODEL.md, docs/SLO_SEMANTICS.md, and
+ADR-0002 for the specification these types implement.
+
+Rules for this package, enforced by review (not yet by tooling):
+
+- No imports from ``agent_reliability.adapters``.
+- No imports of network, filesystem, or database libraries.
+- No dependency on any specific LLM provider or agent framework.
+- Values are immutable.
+
+The exports of this subpackage are part of the stable 1.0 contract documented
+in docs/GA_CONTRACT.md. They are not re-exported from the
+``agent_reliability`` package root, which exposes only ``__version__``.
+
+M4 evaluator execution/provenance lives in the separate
+``agent_reliability.evaluation`` namespace and depends on this domain, never
+the reverse; see docs/ARCHITECTURE.md.
+"""
+
+from __future__ import annotations
+
+from agent_reliability.domain.error_budget import (
+    BudgetStatus,
+    BurnRate,
+    ErrorBudget,
+    compute_burn_rate,
+    compute_error_budget,
+)
+from agent_reliability.domain.evaluation import EvaluationOutcome
+from agent_reliability.domain.identity import AgentIdentity
+from agent_reliability.domain.runs import AgentRun, RunStatus
+from agent_reliability.domain.sli import (
+    ObservationCounts,
+    RatioResult,
+    UnknownPolicy,
+    compute_ratio,
+)
+from agent_reliability.domain.slo import (
+    ObjectiveDirection,
+    Slo,
+    SloEvaluation,
+    SloStatus,
+    evaluate_slo,
+)
+
+__all__ = [
+    "AgentIdentity",
+    "AgentRun",
+    "BudgetStatus",
+    "BurnRate",
+    "ErrorBudget",
+    "EvaluationOutcome",
+    "ObjectiveDirection",
+    "ObservationCounts",
+    "RatioResult",
+    "RunStatus",
+    "Slo",
+    "SloEvaluation",
+    "SloStatus",
+    "UnknownPolicy",
+    "compute_burn_rate",
+    "compute_error_budget",
+    "compute_ratio",
+    "evaluate_slo",
+]

agent_reliability/domain/error_budget.py

@@ -0,0 +1,153 @@
+"""Error budget and burn rate.
+
+Both are, mechanically, the same computation —
+``observed_bad_fraction / allowed_bad_fraction`` — applied to different
+observation windows: a full window for the cumulative error budget, an
+arbitrary shorter lookback for burn rate (docs/SLO_SEMANTICS.md,
+ADR-0002). ``_consumption`` below is the single shared implementation;
+``compute_error_budget`` and ``compute_burn_rate`` are thin, differently
+-shaped public wrappers around it, so the two can never silently drift
+apart.
+
+Three states are distinguished (``BudgetStatus``), because two
+different situations both prevent an ordinary finite number from being
+produced, and they are not the same situation:
+
+- ``NO_DATA``: no considered observations at all (division is
+  undefined because there is nothing to divide).
+- ``ZERO_TOLERANCE_INTACT`` / ``ZERO_TOLERANCE_EXCEEDED``: data exists,
+  but the SLO's target is 100% (``AT_LEAST``) or 0% (``AT_MOST``), so
+  ``allowed_bad_fraction == 0``. Zero observed bad events against a
+  zero-tolerance budget is well-defined (fully intact); any observed
+  bad event makes the true consumption unbounded, which
+  ``fractions.Fraction`` cannot represent (it has no infinity) — this
+  is reported as its own status with the numeric value left ``None``,
+  rather than smuggled in as a ``float('inf')``.
+
+See ADR-0002 for the full reasoning and alternatives considered.
+"""
+
+from __future__ import annotations
+
+import enum
+from dataclasses import dataclass
+from fractions import Fraction
+
+from agent_reliability.domain.sli import RatioResult
+from agent_reliability.domain.slo import Slo
+
+__all__ = [
+    "BudgetStatus",
+    "BurnRate",
+    "ErrorBudget",
+    "compute_burn_rate",
+    "compute_error_budget",
+]
+
+
+class BudgetStatus(enum.Enum):
+    """Shared status for any bad-fraction-over-allowed-fraction division.
+
+    Used by both ``ErrorBudget`` (cumulative, full-window) and
+    ``BurnRate`` (a lookback window) because they are the same
+    underlying division — see module docstring.
+    """
+
+    MEASURED = "measured"
+    NO_DATA = "no_data"
+    ZERO_TOLERANCE_INTACT = "zero_tolerance_intact"
+    ZERO_TOLERANCE_EXCEEDED = "zero_tolerance_exceeded"
+
+
+def _consumption(
+    allowed_bad_fraction: Fraction, ratio: RatioResult
+) -> tuple[BudgetStatus, Fraction | None]:
+    """The single shared computation behind both public functions below.
+
+    Returns ``(status, value)`` where ``value`` is
+    ``observed_bad_fraction / allowed_bad_fraction`` when that division
+    is well-defined, and ``None`` otherwise (with ``status`` explaining
+    why — see module docstring).
+    """
+    observed_bad_fraction = ratio.fail_ratio
+    if observed_bad_fraction is None:
+        return BudgetStatus.NO_DATA, None
+    if allowed_bad_fraction == 0:
+        if observed_bad_fraction == 0:
+            return BudgetStatus.ZERO_TOLERANCE_INTACT, Fraction(0)
+        return BudgetStatus.ZERO_TOLERANCE_EXCEEDED, None
+    return BudgetStatus.MEASURED, observed_bad_fraction / allowed_bad_fraction
+
+
+@dataclass(frozen=True)
+class ErrorBudget:
+    """The error budget implied by an ``Slo`` over one observation window.
+
+    ``allowed_bad_events`` may be non-integral (e.g. ``4.995`` for a
+    0.5% allowance over 999 considered events) — this is kept exact via
+    ``Fraction`` rather than rounded, per docs/SLO_SEMANTICS.md.
+    ``observed_bad_events`` is always a plain, exact integer count (it is
+    never reconstructed by multiplying a ratio back out).
+
+    ``consumption_ratio`` and ``remaining_fraction`` are ``None`` iff
+    ``status`` is ``NO_DATA`` or ``ZERO_TOLERANCE_EXCEEDED``.
+    ``remaining_fraction`` is not clamped to ``[0, 1]``: a negative value
+    means the budget is exhausted and exceeded, and the magnitude is the
+    size of the breach (docs/SLO_SEMANTICS.md).
+    """
+
+    slo: Slo
+    ratio: RatioResult
+    status: BudgetStatus
+    allowed_bad_fraction: Fraction
+    allowed_bad_events: Fraction
+    observed_bad_events: int
+    consumption_ratio: Fraction | None
+    remaining_fraction: Fraction | None
+
+
+def compute_error_budget(slo: Slo, ratio: RatioResult) -> ErrorBudget:
+    """Compute the error budget implied by ``slo`` over ``ratio``'s window."""
+    allowed_bad_fraction = slo.allowed_bad_fraction
+    status, consumption_ratio = _consumption(allowed_bad_fraction, ratio)
+    remaining_fraction = (
+        None if consumption_ratio is None else Fraction(1) - consumption_ratio
+    )
+    return ErrorBudget(
+        slo=slo,
+        ratio=ratio,
+        status=status,
+        allowed_bad_fraction=allowed_bad_fraction,
+        allowed_bad_events=allowed_bad_fraction * ratio.considered_count,
+        observed_bad_events=ratio.considered_fail_count,
+        consumption_ratio=consumption_ratio,
+        remaining_fraction=remaining_fraction,
+    )
+
+
+@dataclass(frozen=True)
+class BurnRate:
+    """How fast a budget is being consumed over one (typically short)
+    lookback window, relative to the rate that would exactly exhaust it
+    over the SLO's full observation window.
+
+    ``value`` of ``1`` means consuming budget at exactly the sustainable
+    rate; ``> 1`` means faster than sustainable. ``None`` iff ``status``
+    is ``NO_DATA`` or ``ZERO_TOLERANCE_EXCEEDED`` — see module docstring.
+
+    ``ratio`` represents whatever lookback period the caller chose; this
+    function has no notion of window length or time itself (see
+    docs/SLO_SEMANTICS.md — multi-window/time-aware alerting is deferred).
+    """
+
+    slo: Slo
+    ratio: RatioResult
+    status: BudgetStatus
+    value: Fraction | None
+
+
+def compute_burn_rate(slo: Slo, ratio: RatioResult) -> BurnRate:
+    """Compute the burn rate implied by ``slo`` over ``ratio``'s (lookback)
+    window."""
+    status, value = _consumption(slo.allowed_bad_fraction, ratio)
+    return BurnRate(slo=slo, ratio=ratio, status=status, value=value)

agent_reliability/domain/evaluation.py

@@ -0,0 +1,36 @@
+"""Evaluation outcomes.
+
+An ``EvaluationOutcome`` is the categorical result of assessing some
+property of an agent run (task success, correctness, policy compliance,
+...). It is deliberately a three-valued enum, not a ``bool``: missing or
+inconclusive evidence (``UNKNOWN``) is a distinct, first-class outcome,
+never collapsed into ``PASS`` or ``FAIL`` and never represented as
+``None`` (see docs/DOMAIN_MODEL.md).
+
+This module intentionally does not define the richer M4 ``EvaluationResult``
+or a separate "reliability observation" wrapper type. For the ratio
+mathematics in ``sli.py``, an eligible observation *is* its outcome — see
+ADR-0002. Evaluator execution and provenance live one layer above in
+``agent_reliability.evaluation``; the mathematical domain does not depend on
+them.
+"""
+
+from __future__ import annotations
+
+import enum
+
+__all__ = ["EvaluationOutcome"]
+
+
+class EvaluationOutcome(enum.Enum):
+    """The categorical result of one evaluation.
+
+    ``PASS`` and ``FAIL`` are never inferred from a quantitative score;
+    thresholding a score into one of these outcomes is evaluator/policy
+    behavior that lives outside this domain kernel (see
+    docs/DOMAIN_MODEL.md, "Evaluation score" vs. "Evaluation result").
+    """
+
+    PASS = "pass"
+    FAIL = "fail"
+    UNKNOWN = "unknown"

agent_reliability/domain/identity.py

@@ -0,0 +1,43 @@
+"""Agent identity.
+
+Deliberately minimal: models only what M1 needs, per docs/DOMAIN_MODEL.md.
+The library intentionally makes no cross-version identity inference:
+``AgentIdentity`` has ordinary structural (dataclass) equality only.
+"""
+
+from __future__ import annotations
+
+from dataclasses import dataclass
+
+__all__ = ["AgentIdentity"]
+
+
+@dataclass(frozen=True)
+class AgentIdentity:
+    """Identifies what agent, at what version, produced a run.
+
+    ``agent_id``, ``name``, and ``version`` are required: without a
+    version an agent's runs cannot be attributed to a specific build for
+    reliability comparison, and without a stable ``agent_id`` there is
+    no key to compare versions of "the same" agent against later.
+
+    ``environment`` is optional metadata (e.g. "production", "staging").
+    It intentionally has no default — a missing value stays missing
+    rather than silently defaulting to a guess like "production", which
+    could mislabel data whose environment the caller genuinely did not
+    specify. Whether/how ``environment`` should scope or filter an SLI's
+    population of runs is not decided at M1.
+    """
+
+    agent_id: str
+    name: str
+    version: str
+    environment: str | None = None
+
+    def __post_init__(self) -> None:
+        if not self.agent_id:
+            raise ValueError("AgentIdentity.agent_id must not be empty")
+        if not self.name:
+            raise ValueError("AgentIdentity.name must not be empty")
+        if not self.version:
+            raise ValueError("AgentIdentity.version must not be empty")