agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/autoinstrument.py

@@ -0,0 +1,145 @@
+from __future__ import annotations
+
+import importlib.util
+import os
+import warnings
+from dataclasses import dataclass, field
+from typing import Any
+
+# Framework integrations AgentLoop ships helpers for, mapped to the import name
+# used to detect whether that framework's SDK is installed.
+_DETECTABLE_INTEGRATIONS: dict[str, str] = {
+    "openai": "openai",
+    "langgraph": "langgraph",
+    "crewai": "crewai",
+}
+
+# Integrations that are not detectable as an installed Python package because
+# instrumentation happens outside this process, mapped to guidance.
+_NON_PYTHON_INTEGRATIONS: dict[str, str] = {
+    "vercel_ai_sdk": "JavaScript-side; call trace_from_vercel_ai_events(...) on exported telemetry",
+}
+
+
+@dataclass
+class DetectionResult:
+    """Which framework integrations are available in the current environment.
+
+    This is a *detection* result. It reports whether each integration's SDK is
+    importable in this process; it does not wrap, patch, or register anything.
+    Instrument an application by applying the helpers in
+    :mod:`agentloop.integrations` (or the generic decorator SDK) from that
+    application's own startup.
+    """
+
+    available: list[str] = field(default_factory=list)
+    unavailable: dict[str, str] = field(default_factory=dict)
+
+    @property
+    def ok(self) -> bool:
+        return bool(self.available)
+
+    def to_dict(self) -> dict[str, Any]:
+        return {"ok": self.ok, "available": self.available, "unavailable": self.unavailable}
+
+
+def _module_available(module_name: str) -> bool:
+    """Return True if ``module_name`` is importable, without importing it."""
+    try:
+        return importlib.util.find_spec(module_name) is not None
+    except (ImportError, ValueError):
+        return False
+
+
+def detect_integrations() -> DetectionResult:
+    """Detect which AgentLoop framework integrations are available here.
+
+    Detection only checks package availability with
+    :func:`importlib.util.find_spec` and never imports the framework, so it is
+    cheap and side-effect free and safe to run at application startup. The result
+    is deterministic for a given environment, so repeated calls return equal
+    results.
+
+    This function does not enable instrumentation. To record traces, apply the
+    helpers in :mod:`agentloop.integrations` (for example
+    ``instrument_openai_client``) or the generic decorator SDK from the
+    application process you want to trace.
+    """
+
+    result = DetectionResult()
+
+    for name, module_name in _DETECTABLE_INTEGRATIONS.items():
+        if _module_available(module_name):
+            result.available.append(name)
+        else:
+            result.unavailable[name] = "package not installed"
+
+    result.unavailable.update(_NON_PYTHON_INTEGRATIONS)
+    return result
+
+
+# Backwards-compatible aliases. The previous names implied that calling this
+# enabled instrumentation, which it never did.
+InstrumentationResult = DetectionResult
+
+
+def auto_instrument(**kwargs: Any) -> DetectionResult:
+    """Deprecated alias for :func:`detect_integrations`.
+
+    The old name and its ``enabled`` field wrongly implied that calling it
+    activated instrumentation; it only ever detected installed frameworks. Use
+    :func:`detect_integrations` instead and apply
+    :mod:`agentloop.integrations` helpers from your application startup. Any
+    keyword arguments are ignored and were never used.
+    """
+
+    warnings.warn(
+        "auto_instrument() never enabled instrumentation; it only detects "
+        "installed frameworks. It is deprecated: use detect_integrations() and "
+        "apply agentloop.integrations helpers from your application startup.",
+        DeprecationWarning,
+        stacklevel=2,
+    )
+    return detect_integrations()
+
+
+def instrument(target: Any, *, integration: str, enabled: bool | None = None) -> Any:
+    """Enable a supported adapter on an explicitly supplied application object.
+
+    This startup call is required even when environment configuration is used.
+    False disables installation on this call; it does not unwrap an object that
+    was previously instrumented. Execution boundaries remain host-owned.
+    """
+    if integration not in {"openai", "langgraph_builder", "crewai_task"}:
+        raise ValueError("integration must be openai, langgraph_builder or crewai_task")
+    if enabled is None:
+        setting = os.getenv("AGENTLOOP_INSTRUMENTATION_ENABLED", "true").strip().lower()
+        if setting not in {"true", "false", "1", "0"}:
+            raise ValueError("AGENTLOOP_INSTRUMENTATION_ENABLED must be true/false/1/0")
+        enabled = setting in {"true", "1"}
+    if type(enabled) is not bool:
+        raise TypeError("enabled must be boolean or None")
+    if not enabled:
+        return target
+    if integration == "openai":
+        from agentloop.integrations.openai import instrument_openai_client
+
+        responses = getattr(target, "responses", None)
+        completions = getattr(getattr(target, "chat", None), "completions", None)
+        if not any(
+            callable(getattr(resource, "create", None)) for resource in (responses, completions)
+        ):
+            raise TypeError("OpenAI-like client exposes no supported create method")
+        return instrument_openai_client(target)
+    if integration == "langgraph_builder":
+        from agentloop.integrations.langgraph import instrument_state_graph
+
+        return instrument_state_graph(target)
+    from agentloop.integrations.crewai import instrument_task
+
+    if not any(
+        callable(getattr(target, name, None))
+        for name in ("execute", "execute_sync", "execute_async", "run")
+    ):
+        raise TypeError("CrewAI-like task exposes no supported execution method")
+    return instrument_task(target)

agentloop/batch_experiments.py

@@ -0,0 +1,295 @@
+"""Bounded caller-provided batch candidates with retained per-item outcomes."""
+
+from __future__ import annotations
+
+from dataclasses import asdict, dataclass
+from time import perf_counter
+
+from agentloop.budget_types import Reservation
+from agentloop.context_experiments import _close_invalid, _deadline, _path, _sum_usage, _value
+from agentloop.context_types import EXACT_COUNTS, ContextTokenCount, identifier, synchronous
+from agentloop.experiment_observations import observation
+from agentloop.experiment_types import (
+    ExperimentResult,
+    ExperimentRunner,
+    canonical,
+    fingerprint,
+    reference,
+)
+from agentloop.structural_types import (
+    STRUCTURAL_KEY,
+    BatchItem,
+    BatchRequest,
+    BatchResult,
+    StageReference,
+    publish_structural_observations,
+)
+from agentloop.studies import summarize_values
+from agentloop.tracer import current_trace
+from agentloop.workflows import StageInfo, trace_operation
+
+
+@dataclass(frozen=True)
+class BatchConstraints:
+    batch_size: int
+    provider_max_items: int
+    provider_max_payload_bytes: int
+    provider_ref: str
+    max_items: int = 10000
+    max_batches: int = 128
+    provider_max_input_tokens: int | None = None
+
+    def __post_init__(self):
+        for name in (
+            "batch_size",
+            "provider_max_items",
+            "provider_max_payload_bytes",
+            "max_items",
+            "max_batches",
+        ):
+            if type(getattr(self, name)) is not int or getattr(self, name) < 1:
+                raise ValueError("batch bounds must be positive integers")
+        if (
+            self.batch_size > self.provider_max_items
+            or self.provider_max_items > 1024
+            or self.max_items > 10000
+            or self.max_batches > 1000
+        ):
+            raise ValueError("batch configuration exceeds declared bounds")
+        if self.provider_max_input_tokens is not None and (
+            type(self.provider_max_input_tokens) is not int or self.provider_max_input_tokens < 1
+        ):
+            raise ValueError("input token limit must be a positive integer")
+        reference(self.provider_ref, "provider constraint reference")
+
+
+def batching_experiment_runner(
+    candidate_id,
+    invoke_batch,
+    *,
+    stage,
+    items_path,
+    constraints,
+    implementation_ref,
+    configuration_ref,
+    why_ref,
+    reservation=None,
+    token_counter=None,
+    counter_ref=None,
+    on_item_failure="fail_task",
+):
+    if (
+        type(stage) is not StageReference
+        or type(constraints) is not BatchConstraints
+        or not synchronous(invoke_batch)
+    ):
+        raise ValueError(
+            "batch experiment requires a stage, constraints and trusted batch implementation"
+        )
+    path = _path(items_path)
+    for name, value in (
+        ("implementation_ref", implementation_ref),
+        ("configuration_ref", configuration_ref),
+        ("why_ref", why_ref),
+    ):
+        reference(value, name)
+    if constraints.provider_max_input_tokens is not None and (
+        not synchronous(token_counter) or counter_ref is None
+    ):
+        raise ValueError("token-constrained batches require a declared exact counter")
+    if counter_ref is not None:
+        identifier(counter_ref, "counter_ref")
+    if on_item_failure not in {"fail_task", "return_partial"}:
+        raise ValueError("unknown batch failure semantics")
+    config = {
+        "schema_version": "1.0",
+        "stage": stage.declaration(),
+        "items_path": path,
+        "constraints": asdict(constraints),
+        "implementation_ref": implementation_ref,
+        "configuration_ref": configuration_ref,
+        "why_ref": why_ref,
+        "counter_ref": counter_ref,
+        "on_item_failure": on_item_failure,
+    }
+    config_hash = fingerprint(config)
+    source = "batch-config:" + config_hash
+
+    def invoke(request):
+        trace = current_trace()
+        if trace is None:
+            raise ValueError("batch candidate requires an active experiment")
+        receipt = {
+            "schema_version": "1.0",
+            "family": "batching",
+            "config_hash": config_hash,
+            "stage": stage.declaration(),
+            "why_ref": why_ref,
+            "constraints": asdict(constraints),
+            "state": "preparing",
+            "batches": [],
+            "items_planned": None,
+        }
+        trace.metadata[STRUCTURAL_KEY] = receipt
+        completed, failed, unknown, invoked, latencies, usages, outputs = 0, 0, 0, 0, [], [], []
+        items = None
+        try:
+            items = _value(request.inputs, path)
+            if not isinstance(items, list) or len(items) > constraints.max_items:
+                raise ValueError("batch input must be a bounded item list")
+            receipt["items_planned"] = len(items)
+            planned_batches = (len(items) + constraints.batch_size - 1) // constraints.batch_size
+            if planned_batches > constraints.max_batches:
+                raise ValueError("planned batches exceed the execution bound")
+            for offset in range(0, len(items), constraints.batch_size):
+                batch = tuple(
+                    BatchItem(f"item_{index:06d}", items[index])
+                    for index in range(offset, min(offset + constraints.batch_size, len(items)))
+                )
+                batch_id = f"batch_{offset // constraints.batch_size:06d}"
+                record = {
+                    "batch_id": batch_id,
+                    "item_ids": [item.item_id for item in batch],
+                    "input_hashes": [fingerprint(item.inputs) for item in batch],
+                    "status": "pending",
+                    "latency_ms": None,
+                    "outcomes": [],
+                    "usage": None,
+                }
+                receipt["batches"].append(record)
+                encoded = canonical(
+                    [{"item_id": item.item_id, "inputs": item.inputs} for item in batch]
+                )
+                if len(encoded.encode("utf-8")) > constraints.provider_max_payload_bytes:
+                    record["status"] = "provider_constraint_rejected"
+                    raise ValueError("batch exceeds declared provider payload bound")
+                _deadline(request)
+                if constraints.provider_max_input_tokens is not None:
+                    count = token_counter(batch)
+                    if (
+                        type(count) is not ContextTokenCount
+                        or count.provenance not in EXACT_COUNTS
+                        or count.reference != counter_ref
+                        or count.value is None
+                        or count.value > constraints.provider_max_input_tokens
+                    ):
+                        record["status"] = "provider_constraint_rejected"
+                        _close_invalid(count)
+                        raise ValueError("batch input token bound is unavailable or exceeded")
+                    record["input_token_count"] = count.to_dict()
+                _deadline(request)
+                began = perf_counter()
+                invoked += 1
+                try:
+                    with trace_operation(
+                        stage.stage_id,
+                        kind=stage.kind,
+                        stage=StageInfo(
+                            stage.stage_id,
+                            stage.version,
+                            kind=stage.kind,
+                            input_ref=request.input_ref,
+                        ),
+                        metadata={
+                            "batch_id": batch_id,
+                            "batch_size": len(batch),
+                            "constraint_ref": constraints.provider_ref,
+                        },
+                    ):
+                        result = invoke_batch(BatchRequest(batch_id, batch, request))
+                    if type(result) is not BatchResult:
+                        _close_invalid(result)
+                        raise ValueError("batch implementation must return BatchResult")
+                    record["usage"] = result.usage.to_dict()
+                    usages.append(result.usage)
+                    expected = {item.item_id for item in batch}
+                    if (
+                        len(result.items) != len(expected)
+                        or {item.item_id for item in result.items} != expected
+                    ):
+                        raise ValueError(
+                            "batch outcomes must match every planned item exactly once"
+                        )
+                    by_id = {item.item_id: item for item in result.items}
+                    for item in batch:
+                        outcome = by_id[item.item_id]
+                        record["outcomes"].append(
+                            {
+                                "item_id": item.item_id,
+                                "status": outcome.status,
+                                "error_code": outcome.error_code,
+                                "output_hash": fingerprint(outcome.output)
+                                if outcome.status == "completed"
+                                else None,
+                            }
+                        )
+                        outputs.append(outcome.output if outcome.status == "completed" else None)
+                        completed += outcome.status == "completed"
+                        failed += outcome.status == "failed"
+                        unknown += outcome.status == "unknown"
+                    record["status"] = (
+                        "completed"
+                        if all(item.status == "completed" for item in result.items)
+                        else "partial"
+                    )
+                    if record["status"] == "partial" and on_item_failure == "fail_task":
+                        raise ValueError("partial batch failure requires task failure")
+                except BaseException:
+                    if record["status"] == "pending":
+                        record["status"] = "failed"
+                        record["outcomes"] = [
+                            {
+                                "item_id": item.item_id,
+                                "status": "unknown",
+                                "error_code": "batch_error",
+                                "output_hash": None,
+                            }
+                            for item in batch
+                        ]
+                        unknown += len(batch)
+                    raise
+                finally:
+                    record["latency_ms"] = (perf_counter() - began) * 1000
+                    latencies.append(record["latency_ms"])
+            receipt["state"] = "completed"
+            if not usages:
+                return ExperimentResult([])
+            return ExperimentResult(outputs, usage=_sum_usage(usages))
+        except BaseException:
+            receipt["state"] = "failed"
+            raise
+        finally:
+            receipt["latency_distribution_ms"] = summarize_values(latencies)
+            planned = len(items) if isinstance(items, list) else None
+            remaining = None if planned is None else planned - completed - failed - unknown
+            publish_structural_observations(
+                receipt,
+                {
+                    "batch_state": observation(receipt["state"], unit="state", source_ref=source),
+                    "batch_size": observation(
+                        constraints.batch_size, kind="declared", unit="items", source_ref=source
+                    ),
+                    "batch_invocations": observation(invoked, unit="calls", source_ref=source),
+                    "batch_items_planned": observation(planned, unit="items", source_ref=source),
+                    "batch_items_completed": observation(
+                        completed, unit="completed_items", source_ref=source
+                    ),
+                    "batch_items_failed": observation(failed, unit="items", source_ref=source),
+                    "batch_items_unknown": observation(unknown, unit="items", source_ref=source),
+                    "batch_items_not_started": observation(
+                        remaining, unit="items", source_ref=source
+                    ),
+                    "batch_latency_p95_ms": observation(
+                        receipt["latency_distribution_ms"]["p95"], unit="ms", source_ref=source
+                    ),
+                },
+            )
+
+    return ExperimentRunner(
+        candidate_id,
+        "1.0",
+        "agentloop.batching:1.0",
+        "sha256:" + config_hash,
+        invoke,
+        reservation=reservation or Reservation(),
+    )

agentloop/batching.py

@@ -0,0 +1,95 @@
+"""Model-call groups that do not contradict recorded causal dependencies."""
+
+from __future__ import annotations
+
+from collections import deque
+from collections.abc import Mapping
+
+from agentloop.workflow_types import stage_summary
+
+
+def model_batch_groups(graph):
+    """Return hypotheses; absent dependencies never prove output independence.
+
+    Index dependencies once, reject unresolved/cyclic ancestry, then inspect each
+    candidate group's ancestor closure without recursion or all-pairs searches.
+    Inferred sequence edges describe ordering, not causal dependencies.
+    """
+    by_id = {node.node_id: node for node in graph.nodes}
+    if len(by_id) != len(graph.nodes):
+        return []
+    dependencies = {identity: set() for identity in by_id}
+    invalid = set()
+    groups = {}
+    for node in graph.nodes:
+        metadata = node.metadata
+        if not isinstance(metadata, Mapping):
+            invalid.add(node.node_id)
+            metadata = {}
+        required = metadata.get("depends_on", [])
+        if not isinstance(required, list) or any(
+            not isinstance(identity, str) or not identity for identity in required
+        ):
+            invalid.add(node.node_id)
+        else:
+            dependencies[node.node_id].update(required)
+        stage = stage_summary(node)
+        if stage is not None:
+            if (
+                stage.get("schema_status") != "supported"
+                or stage.get("dependency_status") == "invalid"
+            ):
+                invalid.add(node.node_id)
+            elif stage.get("dependencies_declared"):
+                dependencies[node.node_id].update(stage["depends_on"])
+        if node.parent_id is not None:
+            dependencies[node.node_id].add(node.parent_id)
+        if node.operation_kind == "model":
+            groups.setdefault((node.name, node.model, node.parent_id), []).append(node.node_id)
+    if not any(len(identities) >= 3 for identities in groups.values()):
+        return []
+    for edge in graph.edges:
+        if edge.kind in {"dependency", "depends_on", "parent"} and edge.target in by_id:
+            dependencies[edge.target].add(edge.source)
+
+    # Removing roots leaves cycles and their descendants unresolved. Invalid or
+    # missing ancestors propagate to descendants as the valid portion is visited.
+    successors = {identity: [] for identity in by_id}
+    indegrees = {}
+    for identity, required in dependencies.items():
+        indegrees[identity] = len(required)
+        for ancestor in required:
+            if ancestor in successors:
+                successors[ancestor].append(identity)
+            else:
+                invalid.add(identity)
+    ready = deque(identity for identity, count in indegrees.items() if not count)
+    visited = set()
+    while ready:
+        identity = ready.popleft()
+        visited.add(identity)
+        for successor in successors[identity]:
+            if identity in invalid:
+                invalid.add(successor)
+            indegrees[successor] -= 1
+            if indegrees[successor] == 0:
+                ready.append(successor)
+    invalid.update(by_id.keys() - visited)
+
+    eligible = []
+    for (name, _, _), identities in groups.items():
+        members = set(identities)
+        if len(members) < 3 or members & invalid:
+            continue
+        pending = [ancestor for identity in identities for ancestor in dependencies[identity]]
+        ancestors = set()
+        while pending:
+            ancestor = pending.pop()
+            if ancestor in members:
+                break
+            if ancestor not in ancestors:
+                ancestors.add(ancestor)
+                pending.extend(dependencies[ancestor])
+        else:
+            eligible.append((name, [by_id[identity] for identity in identities]))
+    return eligible