agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/reuse_experiments.py

@@ -0,0 +1,247 @@
+"""Exact-key reuse from a frozen prior-result snapshot; no cache service."""
+
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+
+from agentloop.ablation_protocol import timestamp
+from agentloop.budget_types import ResourceUsage
+from agentloop.context_experiments import _path, _value
+from agentloop.context_types import identifier
+from agentloop.experiment_observations import observation, record_experiment_observations
+from agentloop.experiment_types import (
+    ExperimentResult,
+    ExperimentRunner,
+    canonical,
+    fingerprint,
+    reference,
+)
+from agentloop.tracer import current_trace
+
+REUSE_KEY = "agentloop.reuse_experiment"
+
+
+@dataclass(frozen=True, init=False)
+class ReuseEntry:
+    _json: str = field(repr=False)
+    _result: ExperimentResult = field(repr=False)
+
+    def __init__(
+        self,
+        key,
+        output,
+        *,
+        key_version=None,
+        implementation_ref=None,
+        implementation_version=None,
+        configuration_ref=None,
+        data_ref=None,
+        invalidation_epoch=None,
+        source_ref=None,
+        created_at=None,
+        expires_at=None,
+    ):
+        for name, value in (
+            ("key_version", key_version),
+            ("implementation_ref", implementation_ref),
+            ("implementation_version", implementation_version),
+            ("configuration_ref", configuration_ref),
+            ("data_ref", data_ref),
+            ("invalidation_epoch", invalidation_epoch),
+            ("source_ref", source_ref),
+        ):
+            if value is not None:
+                reference(value, name)
+        for value in (created_at, expires_at):
+            if value is not None:
+                timestamp(value)
+        result = ExperimentResult(output)
+        data = {
+            "key_hash": fingerprint(key),
+            "output_hash": fingerprint(result.output),
+            "key_version": key_version,
+            "implementation_ref": implementation_ref,
+            "implementation_version": implementation_version,
+            "configuration_ref": configuration_ref,
+            "data_ref": data_ref,
+            "invalidation_epoch": invalidation_epoch,
+            "source_ref": source_ref,
+            "created_at": created_at,
+            "expires_at": expires_at,
+        }
+        object.__setattr__(self, "_json", canonical(data))
+        object.__setattr__(self, "_result", result)
+
+    @property
+    def output(self):
+        return self._result.output
+
+    def declaration(self):
+        import json
+
+        return json.loads(self._json)
+
+
+def result_reuse_runner(
+    candidate_id,
+    backend,
+    *,
+    entries,
+    key_paths,
+    key_version,
+    data_ref,
+    invalidation_epoch,
+    as_of,
+    on_unavailable="recompute",
+    reservation=None,
+):
+    if (
+        type(backend) is not ExperimentRunner
+        or not isinstance(entries, (list, tuple))
+        or len(entries) > 1000
+        or any(type(entry) is not ReuseEntry for entry in entries)
+    ):
+        raise ValueError("reuse requires an explicit backend and bounded frozen entry snapshot")
+    if not isinstance(key_paths, (list, tuple)) or not 1 <= len(key_paths) <= 32:
+        raise ValueError("reuse keys require explicit bounded input paths")
+    paths = tuple(_path(path) for path in key_paths)
+    if len(set(paths)) != len(paths):
+        raise ValueError("reuse key paths must be unique")
+    identifier(key_version, "key_version")
+    reference(data_ref, "data_ref")
+    reference(invalidation_epoch, "invalidation_epoch")
+    instant = timestamp(as_of)
+    if on_unavailable not in {"recompute", "error"}:
+        raise ValueError("unknown reuse fallback policy")
+    entries = tuple(entries)
+    declarations = [entry.declaration() for entry in entries]
+    if sum(len(canonical(entry.output).encode("utf-8")) for entry in entries) > 10_000_000:
+        raise ValueError("reuse snapshot exceeds ten million output bytes")
+    config = {
+        "schema_version": "1.0",
+        "backend": backend.declaration(),
+        "key_paths": paths,
+        "key_version": key_version,
+        "data_ref": data_ref,
+        "invalidation_epoch": invalidation_epoch,
+        "as_of": as_of,
+        "on_unavailable": on_unavailable,
+        "entries": declarations,
+        "snapshot_semantics": "read_only_prior_results_no_implicit_fill",
+    }
+    config_hash = fingerprint(config)
+    source = "reuse-config:" + config_hash
+    indexed = {}
+    for entry, declared in zip(entries, declarations):
+        indexed.setdefault(declared["key_hash"], []).append((entry, declared))
+    required = {
+        "key_version": key_version,
+        "implementation_ref": backend.implementation_ref,
+        "implementation_version": backend.version,
+        "configuration_ref": backend.configuration_ref,
+        "data_ref": data_ref,
+        "invalidation_epoch": invalidation_epoch,
+    }
+
+    def invoke(request):
+        if backend.declaration() != config["backend"]:
+            raise ValueError("reuse backend binding changed after configuration")
+        trace = current_trace()
+        if trace is None:
+            raise ValueError("reuse candidate requires an active experiment")
+        receipt = {
+            "schema_version": "1.0",
+            "config_hash": config_hash,
+            "key_hash": None,
+            "state": "key_unavailable",
+            "entry_refs": [],
+            "fallback": False,
+            "as_of": as_of,
+        }
+        trace.metadata[REUSE_KEY] = receipt
+        valid = []
+        try:
+            payload = request.inputs
+            key = [_value(payload, path) for path in paths]
+        except ValueError:
+            matches = []
+        else:
+            receipt["key_hash"] = fingerprint(key)
+            matches = indexed.get(receipt["key_hash"], [])
+            receipt["state"] = "miss"
+        states = []
+        for entry, declared in matches:
+            receipt["entry_refs"].append(
+                {
+                    "source_ref": declared["source_ref"],
+                    "entry_hash": fingerprint(declared),
+                    "output_hash": declared["output_hash"],
+                }
+            )
+            if any(
+                declared[key] is None
+                for key in (*required, "source_ref", "created_at", "expires_at")
+            ):
+                states.append("unknown_evidence")
+            elif any(declared[key] != value for key, value in required.items()):
+                states.append("invalidated")
+            elif timestamp(declared["created_at"]) >= timestamp(declared["expires_at"]):
+                states.append("invalid_lifetime")
+            elif timestamp(declared["created_at"]) > instant:
+                states.append("future_entry")
+            elif timestamp(declared["expires_at"]) <= instant:
+                states.append("stale")
+            else:
+                valid.append((entry, declared))
+        if valid:
+            newest = max(timestamp(declared["created_at"]) for _, declared in valid)
+            selected = [
+                (entry, declared)
+                for entry, declared in valid
+                if timestamp(declared["created_at"]) == newest
+            ]
+            if len({declared["output_hash"] for _, declared in selected}) == 1:
+                chosen = min(selected, key=lambda item: fingerprint(item[1]))
+                receipt.update(state="hit", selected_entry_hash=fingerprint(chosen[1]))
+            else:
+                receipt["state"] = "ambiguous"
+        elif states:
+            receipt["state"] = sorted(states)[0]
+        receipt["unusable_entry_states"] = sorted(states)
+        hit = receipt["state"] == "hit"
+        expired = not hit and on_unavailable == "recompute" and request.remaining_s <= 0
+        receipt["fallback"] = not hit and on_unavailable == "recompute" and not expired
+        record_experiment_observations(
+            {
+                "cache_hit": observation(hit, unit="boolean", source_ref=source),
+                "cache_state": observation(receipt["state"], unit="state", source_ref=source),
+                "cache_fallback_invoked": observation(
+                    receipt["fallback"], unit="boolean", source_ref=source
+                ),
+                "cache_fill_cost_included": observation(
+                    False, kind="declared", unit="boolean", source_ref=source
+                ),
+            }
+        )
+        if hit:
+            # No provider token work is performed by a snapshot hit. Lookup and
+            # prior fill operating costs remain unknown/excluded, not free.
+            # token_provenance is an accounting enum, not a credential.
+            return ExperimentResult(
+                chosen[0].output,
+                usage=ResourceUsage(tokens=0, token_provenance="user_supplied", complete=True),  # nosec B106
+            )
+        if expired:
+            raise TimeoutError("reuse fallback deadline expired")
+        if on_unavailable == "error":
+            raise ValueError("reuse evidence is unavailable under the declared key contract")
+        return backend.invoke(request)
+
+    return ExperimentRunner(
+        candidate_id,
+        "1.0",
+        "agentloop.result-reuse:1.0",
+        "sha256:" + config_hash,
+        invoke,
+        reservation=reservation if reservation is not None else backend.reservation,
+    )

agentloop/routing_checks.py

@@ -0,0 +1,116 @@
+"""Conservative checks before an explicitly configured model substitution."""
+
+from __future__ import annotations
+
+from copy import deepcopy
+
+from agentloop.context_types import EXACT_COUNTS, ContextTokenCount
+from agentloop.harness import _invoke_usage_reader
+
+
+def check_route(backend, payload, *, streaming):
+    capabilities = backend.capabilities
+    reasons = []
+    if capabilities.adapter != "json_chat_v1":
+        reasons.append("unsupported_adapter")
+    if not set(payload) <= set(capabilities.parameters):
+        reasons.append("unsupported_parameters")
+    if streaming and (not capabilities.streaming or backend.stream is None):
+        reasons.append("streaming_unsupported")
+    if payload.get("stream", False) is not streaming:
+        reasons.append("stream_mode_mismatch")
+    messages = payload.get("messages")
+    modalities, has_tools = set(), bool(payload.get("tools"))
+    if not isinstance(messages, list) or not messages or len(messages) > 512:
+        reasons.append("invalid_messages")
+        messages = []
+    for message in messages:
+        if (
+            not isinstance(message, dict)
+            or not isinstance(message.get("role"), str)
+            or message.get("role")
+            not in {
+                "system",
+                "developer",
+                "user",
+                "assistant",
+                "tool",
+            }
+        ):
+            reasons.append("unsupported_message")
+            continue
+        has_tools = has_tools or message["role"] == "tool" or bool(message.get("tool_calls"))
+        content = message.get("content")
+        if isinstance(content, str):
+            modalities.add("text")
+        elif content is None and message["role"] == "assistant" and message.get("tool_calls"):
+            modalities.add("text")
+        elif isinstance(content, list) and content:
+            for part in content:
+                kind = part.get("type") if isinstance(part, dict) else None
+                mapped = (
+                    {"text": "text", "image_url": "image", "input_audio": "audio"}.get(kind)
+                    if isinstance(kind, str)
+                    else None
+                )
+                if mapped is None:
+                    reasons.append("unsupported_content")
+                else:
+                    modalities.add(mapped)
+        else:
+            reasons.append("unsupported_content")
+    if not modalities <= set(capabilities.modalities):
+        reasons.append("modality_unsupported")
+    if has_tools and not capabilities.tool_calling:
+        reasons.append("tools_unsupported")
+    if payload.get("tools") is not None and (
+        not isinstance(payload["tools"], list)
+        or not payload["tools"]
+        or any(
+            not isinstance(tool, dict)
+            or tool.get("type") != "function"
+            or not isinstance(tool.get("function"), dict)
+            for tool in payload["tools"]
+        )
+    ):
+        reasons.append("unsupported_tool_format")
+    response = payload.get("response_format", {"type": "text"})
+    mode = response.get("type") if isinstance(response, dict) else None
+    if not isinstance(mode, str) or mode not in capabilities.output_modes:
+        reasons.append("output_mode_unsupported")
+    if mode == "json_schema" and (
+        not isinstance(response.get("json_schema"), dict)
+        or not isinstance(response["json_schema"].get("schema"), dict)
+    ):
+        reasons.append("invalid_output_schema")
+    bounds = [payload[key] for key in ("max_tokens", "max_completion_tokens") if key in payload]
+    output_bound = (
+        bounds[0] if len(bounds) == 1 and type(bounds[0]) is int and bounds[0] > 0 else None
+    )
+    if output_bound is None:
+        reasons.append("output_bound_unavailable")
+    count = ContextTokenCount()
+    count_attempted = not reasons
+    if count_attempted:
+        try:
+            value = _invoke_usage_reader(backend.token_counter, deepcopy(payload))
+            if type(value) is ContextTokenCount:
+                count = value
+            else:
+                reasons.append("invalid_token_count")
+        except Exception:
+            reasons.append("token_counter_failed")
+        if count.provenance not in EXACT_COUNTS or count.value is None:
+            reasons.append("context_count_unverified")
+        elif output_bound is not None and count.value + output_bound > capabilities.context_window:
+            reasons.append("context_limit")
+    return {
+        "supported": not reasons,
+        "reasons": sorted(set(reasons)),
+        "capability_reference": capabilities.reference,
+        "adapter": capabilities.adapter,
+        "input_tokens": count.to_dict(),
+        "token_count_attempted": count_attempted,
+        "output_token_bound": output_bound,
+        "context_window": capabilities.context_window,
+    }

agentloop/routing_types.py

@@ -0,0 +1,241 @@
+"""Owned requests and explicit provider/model capability declarations."""
+
+from __future__ import annotations
+
+import json
+from dataclasses import asdict, dataclass, field
+from typing import Any, Callable
+
+from agentloop.budget_types import DispatchOptions, ResourceUsage
+from agentloop.context_types import ContextTokenCount, canonical, identifier, synchronous
+
+ROUTING_VERSION = "1.0"
+ROUTING_KEY = "agentloop.model_routing"
+FAILURE_KINDS = frozenset({"rate_limit", "unavailable", "timeout", "provider_error"})
+
+
+def model_name(value):
+    if (
+        not isinstance(value, str)
+        or not value
+        or len(value) > 512
+        or any(ord(char) < 32 for char in value)
+    ):
+        raise ValueError("model identity must be a bounded nonempty string")
+
+
+class RoutingError(ValueError):
+    """A fixed failure category without provider payloads."""
+
+    def __init__(self, reason):
+        allowed = {
+            "invalid_request",
+            "unsupported_route",
+            "request_model_mismatch",
+            "invalid_response",
+            "inspection_failed",
+            "evidence_limit",
+            "invalid_trace_evidence",
+            "recursive_route",
+            "unsupported_stream",
+            "context_adapter_nesting",
+        }
+        self.reason_code = (
+            reason if isinstance(reason, str) and reason in allowed else "invalid_request"
+        )
+        super().__init__(f"Model routing failed: {self.reason_code}")
+
+
+@dataclass(frozen=True)
+class ModelIdentity:
+    provider: str
+    model: str
+
+    def __post_init__(self):
+        identifier(self.provider, "provider")
+        model_name(self.model)
+
+    def to_dict(self):
+        return asdict(self)
+
+
+@dataclass(frozen=True)
+class ModelCapabilities:
+    reference: str
+    context_window: int
+    parameters: tuple[str, ...] = ("model", "messages", "max_tokens")
+    output_modes: tuple[str, ...] = ("text",)
+    modalities: tuple[str, ...] = ("text",)
+    tool_calling: bool = False
+    streaming: bool = False
+    adapter: str = "json_chat_v1"
+
+    def __post_init__(self):
+        identifier(self.reference, "capability reference")
+        if type(self.context_window) is not int or self.context_window < 1:
+            raise ValueError("context_window must be positive")
+        for name, allowed in (
+            ("output_modes", {"text", "json_object", "json_schema"}),
+            ("modalities", {"text", "image", "audio"}),
+            ("parameters", None),
+        ):
+            values = getattr(self, name)
+            if (
+                not isinstance(values, (tuple, list, set, frozenset))
+                or not values
+                or len(values) > 64
+                or any(
+                    not isinstance(value, str) or not value or len(value) > 128 for value in values
+                )
+            ):
+                raise ValueError("capabilities require bounded string collections")
+            if allowed is not None and not set(values) <= allowed:
+                raise ValueError("unsupported capability declaration")
+            object.__setattr__(self, name, tuple(sorted(set(values))))
+        if (
+            type(self.tool_calling) is not bool
+            or type(self.streaming) is not bool
+            or not isinstance(self.adapter, str)
+        ):
+            raise ValueError("invalid adapter capability")
+
+    def to_dict(self):
+        return json.loads(canonical(asdict(self)))
+
+
+@dataclass(frozen=True, init=False)
+class RoutingRequest:
+    _json: str = field(repr=False)
+
+    def __init__(self, payload):
+        try:
+            if not isinstance(payload, dict):
+                raise ValueError()
+            canonical(payload)
+            encoded = json.dumps(
+                payload, ensure_ascii=False, separators=(",", ":"), allow_nan=False
+            )
+            if len(encoded.encode("utf-8")) > 1_000_000:
+                raise ValueError()
+            model_name(payload.get("model"))
+        except (ValueError, TypeError, RecursionError):
+            raise RoutingError("invalid_request") from None
+        object.__setattr__(self, "_json", encoded)
+
+    @property
+    def payload(self):
+        return json.loads(self._json)
+
+
+@dataclass(frozen=True)
+class RouteResponseInfo:
+    usage: ResourceUsage | None = None
+    reported_model: str | None = None
+
+    def __post_init__(self):
+        if self.usage is not None and type(self.usage) is not ResourceUsage:
+            raise ValueError("response usage must be normalized ResourceUsage")
+        if self.reported_model is not None:
+            model_name(self.reported_model)
+
+
+class RouteProviderError(Exception):
+    """Host adapter classification permitting only configured failure fallbacks.
+
+    An SDK adapter may raise this from its original SDK exception. Usage is an
+    exclusive snapshot when available; absent billing is never inferred as zero.
+    """
+
+    def __init__(self, category, *, info=None):
+        if (
+            not isinstance(category, str)
+            or category not in FAILURE_KINDS
+            or (info is not None and type(info) is not RouteResponseInfo)
+        ):
+            raise ValueError("invalid provider failure declaration")
+        self.category = category
+        self.info = info or RouteResponseInfo()
+        super().__init__(f"Routed provider failure: {category}")
+
+
+@dataclass(frozen=True)
+class ModelBackend:
+    identity: ModelIdentity
+    capabilities: ModelCapabilities
+    invoke: Callable[[dict], Any] = field(repr=False, compare=False)
+    token_counter: Callable[[dict], ContextTokenCount] = field(repr=False, compare=False)
+    inspect_response: Callable[[Any], RouteResponseInfo] = field(repr=False, compare=False)
+    stream: Callable[[dict], Any] | None = field(default=None, repr=False, compare=False)
+    dispatch: DispatchOptions = field(default_factory=DispatchOptions)
+
+    def __post_init__(self):
+        if (
+            type(self.identity) is not ModelIdentity
+            or type(self.capabilities) is not ModelCapabilities
+            or type(self.dispatch) is not DispatchOptions
+        ):
+            raise ValueError("backend requires typed identity, capabilities and dispatch")
+        if any(
+            not synchronous(callback)
+            for callback in (self.invoke, self.token_counter, self.inspect_response)
+        ):
+            raise ValueError("backend callbacks must be synchronous")
+        if self.stream is not None and not synchronous(self.stream):
+            raise ValueError("stream factory must be synchronous and return an SDK iterator")
+        if self.capabilities.streaming and self.stream is None:
+            raise ValueError("streaming capability requires an explicit SDK stream factory")
+
+
+@dataclass(frozen=True)
+class RoutingConfig:
+    route_id: str
+    version: str
+    branch_id: str
+    quality_ref: str
+    original: ModelIdentity
+    target: ModelIdentity
+    allowlist: tuple[ModelIdentity, ...]
+    fallbacks: tuple[ModelIdentity, ...] = ()
+    fallback_on: tuple[str, ...] = ()
+    on_unsupported: str = "error"
+    allow_cross_provider: bool = False
+
+    def __post_init__(self):
+        for name in ("route_id", "version", "branch_id", "quality_ref"):
+            identifier(getattr(self, name), name)
+        if type(self.original) is not ModelIdentity or type(self.target) is not ModelIdentity:
+            raise ValueError("route requires typed original and target models")
+        for name, maximum in (("allowlist", 32), ("fallbacks", 2)):
+            values = getattr(self, name)
+            if (
+                not isinstance(values, (list, tuple))
+                or len(values) > maximum
+                or any(type(item) is not ModelIdentity for item in values)
+                or len(set(values)) != len(values)
+            ):
+                raise ValueError("route models must be unique and bounded")
+            object.__setattr__(self, name, tuple(values))
+        if (
+            not {self.original, self.target, *self.fallbacks} <= set(self.allowlist)
+            or self.target in self.fallbacks
+        ):
+            raise ValueError(
+                "every route endpoint must be allowlisted; fallback cannot repeat target"
+            )
+        if not isinstance(self.fallback_on, (list, tuple)) or any(
+            not isinstance(value, str) or value not in FAILURE_KINDS for value in self.fallback_on
+        ):
+            raise ValueError("fallback requires explicit supported failure categories")
+        object.__setattr__(self, "fallback_on", tuple(sorted(set(self.fallback_on))))
+        if (
+            self.on_unsupported not in {"error", "original"}
+            or type(self.allow_cross_provider) is not bool
+        ):
+            raise ValueError("invalid unsupported/cross-provider policy")
+        if not self.allow_cross_provider and any(
+            item.provider != self.original.provider for item in (self.target, *self.fallbacks)
+        ):
+            raise ValueError("cross-provider routing requires explicit configuration")
+
+    def to_dict(self):
+        return json.loads(canonical(asdict(self)))