agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/integrations/sdk.py

@@ -0,0 +1,82 @@
+from __future__ import annotations
+
+from typing import Any
+
+
+def instrument(obj: Any, *, kind: str | None = None, name: str | None = None) -> Any:
+    """Best-effort one-line instrumentation for common agent objects.
+
+    Explicit `kind` values:
+    - `openai`
+    - `crewai_crew`
+    - `crewai_agent`
+    - `crewai_task`
+    - `langgraph_state_graph`
+    - `langgraph_runnable`
+
+    Without `kind`, AgentLoop uses duck-typing to choose the safest wrapper.
+    """
+
+    normalized = kind.lower().replace("-", "_") if kind else None
+
+    if normalized == "openai" or _looks_like_openai_client(obj):
+        from agentloop.integrations.openai import instrument_openai_client
+
+        return instrument_openai_client(obj)
+
+    if normalized == "crewai_crew" or _looks_like_crewai_crew(obj):
+        from agentloop.integrations.crewai import instrument_crew
+
+        return instrument_crew(obj, name=name)
+
+    if normalized == "crewai_agent":
+        from agentloop.integrations.crewai import instrument_agent
+
+        return instrument_agent(obj, name=name)
+
+    if normalized == "crewai_task":
+        from agentloop.integrations.crewai import instrument_task
+
+        return instrument_task(obj, name=name)
+
+    if normalized == "langgraph_state_graph" or _looks_like_state_graph(obj):
+        from agentloop.integrations.langgraph import instrument_state_graph
+
+        return instrument_state_graph(obj)
+
+    if normalized == "langgraph_runnable" or _looks_like_runnable(obj):
+        from agentloop.integrations.langgraph import trace_runnable
+
+        return trace_runnable(
+            obj, name=name or getattr(obj, "name", None) or obj.__class__.__name__
+        )
+
+    raise ValueError(
+        "Could not infer object type for AgentLoop instrumentation. "
+        "Pass kind='openai', 'crewai_crew', 'crewai_agent', 'crewai_task', "
+        "'langgraph_state_graph', or 'langgraph_runnable'."
+    )
+
+
+def _looks_like_openai_client(obj: Any) -> bool:
+    responses = getattr(obj, "responses", None)
+    chat = getattr(obj, "chat", None)
+    completions = getattr(chat, "completions", None) if chat is not None else None
+    return bool(
+        responses is not None
+        and callable(getattr(responses, "create", None))
+        or completions is not None
+        and callable(getattr(completions, "create", None))
+    )
+
+
+def _looks_like_crewai_crew(obj: Any) -> bool:
+    return callable(getattr(obj, "kickoff", None)) or callable(getattr(obj, "kickoff_async", None))
+
+
+def _looks_like_state_graph(obj: Any) -> bool:
+    return callable(getattr(obj, "add_node", None)) and callable(getattr(obj, "compile", None))
+
+
+def _looks_like_runnable(obj: Any) -> bool:
+    return callable(getattr(obj, "invoke", None)) or callable(getattr(obj, "ainvoke", None))

agentloop/integrations/vercel_ai.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import Any
+
+from agentloop.events import AgentEvent, new_event_id, utc_now_iso
+from agentloop.tracer import AgentTrace
+
+
+def _get(data: Any, key: str, default: Any = None) -> Any:
+    if isinstance(data, dict):
+        return data.get(key, default)
+    return getattr(data, key, default)
+
+
+def trace_from_vercel_ai_events(
+    events: list[dict[str, Any]],
+    *,
+    name: str = "vercel_ai_run",
+    run_id: str | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> AgentTrace:
+    """Convert Vercel AI SDK-style telemetry events into an AgentLoop trace.
+
+    The adapter intentionally accepts plain dictionaries so JS/TS apps can POST or
+    export telemetry without importing Python code. Expected fields are flexible:
+    `type`/`event_type`, `name`, `duration_ms`, `model`, `input_tokens`,
+    `output_tokens`, `status`, and `metadata`.
+    """
+
+    trace = AgentTrace(
+        name=name, run_id=run_id, metadata={"integration": "vercel_ai", **(metadata or {})}
+    )
+    for item in events:
+        event_type = str(_get(item, "event_type", _get(item, "type", "model_call")))
+        if event_type in {"text", "generate", "stream", "model"}:
+            event_type = "model_call"
+        elif event_type in {"tool", "tool-call", "step"}:
+            event_type = "tool_call"
+
+        trace.add_event(
+            AgentEvent(
+                event_id=str(_get(item, "event_id", new_event_id())),
+                run_id=trace.run_id,
+                event_type=event_type,
+                name=str(_get(item, "name", event_type)),
+                started_at=str(_get(item, "started_at", utc_now_iso())),
+                ended_at=str(_get(item, "ended_at", utc_now_iso())),
+                duration_ms=float(_get(item, "duration_ms", 0.0) or 0.0),
+                parent_id=_get(item, "parent_id"),
+                model=_get(item, "model"),
+                input_tokens=int(_get(item, "input_tokens", _get(item, "prompt_tokens", 0)) or 0),
+                output_tokens=int(
+                    _get(item, "output_tokens", _get(item, "completion_tokens", 0)) or 0
+                ),
+                input_text=_get(item, "input_text"),
+                output_text=_get(item, "output_text"),
+                status=str(_get(item, "status", "ok")),
+                error=_get(item, "error"),
+                metadata={"integration": "vercel_ai", **(_get(item, "metadata", {}) or {})},
+            )
+        )
+    return trace
+
+
+TYPESCRIPT_SNIPPET = r"""
+// Minimal Vercel AI SDK telemetry bridge for AgentLoop.
+// Push these events to your backend, then convert them with
+// agentloop.integrations.vercel_ai.trace_from_vercel_ai_events(...).
+
+const agentloopEvents: any[] = [];
+
+export function recordAgentLoopEvent(event: any) {
+  agentloopEvents.push({
+    ...event,
+    started_at: event.started_at ?? new Date().toISOString(),
+    ended_at: event.ended_at ?? new Date().toISOString(),
+  });
+}
+
+export function getAgentLoopEvents() {
+  return agentloopEvents;
+}
+"""

agentloop/interoperability/__init__.py

@@ -0,0 +1,5 @@
+"""Internal, dependency-free boundaries for external execution evidence.
+
+These contracts are additive to native traces. Importing this package does not
+discover plugins, load external runtimes, execute task code or contact services.
+"""

agentloop/interoperability/artifacts.py

@@ -0,0 +1,112 @@
+"""Stable local native projections and receipts shared by external importers."""
+
+from __future__ import annotations
+
+import json
+from hashlib import sha256
+from pathlib import Path
+from typing import Any, Iterable
+
+from agentloop.interoperability.contracts import ImportReceipt
+from agentloop.interoperability.validation import (
+    ImportValidationError,
+    indirect_path,
+    relative_reference,
+)
+from agentloop.interventions import canonical_json
+from agentloop.tracer import AgentTrace
+
+
+def native_bytes(trace: AgentTrace) -> bytes:
+    return (
+        json.dumps(trace.to_dict(), indent=2, ensure_ascii=False, allow_nan=False) + "\n"
+    ).encode("utf-8")
+
+
+def write_artifact(root: Path, reference: str, data: bytes) -> None:
+    """Reject indirect paths and conflicting bytes under an explicit output root."""
+    write_artifact_chunks(root, reference, (data,))
+
+
+def output_artifact_path(root: Path, reference: str) -> Path:
+    """Check every generated child reference before creating files or directories."""
+    relative_reference(reference)
+    current = root
+    for part in reference.split("/"):
+        current = current / part
+        try:
+            indirect = indirect_path(current)
+        except FileNotFoundError:
+            indirect = False
+        if indirect:
+            raise ImportValidationError(
+                "unsafe_path", "output", "output references cannot be symlinks"
+            )
+    return current
+
+
+def write_artifact_chunks(root: Path, reference: str, chunks: Iterable[bytes]) -> None:
+    """Write bounded chunks, comparing an existing artifact without loading it all."""
+    target = output_artifact_path(root, reference)
+    target.parent.mkdir(parents=True, exist_ok=True)
+    if target.exists():
+        with target.open("rb") as stream:
+            for chunk in chunks:
+                if stream.read(len(chunk)) != chunk:
+                    raise ImportValidationError(
+                        "output_conflict", "output", "existing artifact has conflicting content"
+                    )
+            if stream.read(1):
+                raise ImportValidationError(
+                    "output_conflict", "output", "existing artifact has conflicting content"
+                )
+        return
+    # Open exclusively before entering cleanup, so a failed competing open
+    # cannot remove an artifact owned by another writer.
+    stream = target.open("xb")
+    try:
+        with stream:
+            for chunk in chunks:
+                stream.write(chunk)
+    except BaseException:
+        # Close before unlinking on Windows. A producer/write error must not
+        # leave a partial immutable artifact that blocks a successful retry.
+        target.unlink(missing_ok=True)
+        raise
+
+
+def write_import_bundle(
+    traces: Iterable[AgentTrace],
+    receipts: Iterable[ImportReceipt],
+    out: str | Path,
+    inventory: dict[str, Any],
+) -> Path:
+    """Verify captured byte hashes before exporting a reproducible import bundle."""
+    traces, receipts = tuple(traces), tuple(receipts)
+    expected = {
+        item["run_id"]: item["trace_sha256"]
+        for receipt in receipts
+        for item in receipt.to_dict()["traces"]
+    }
+    if len({trace.run_id for trace in traces}) != len(traces):
+        raise ImportValidationError(
+            "source_conflict", "native_trace", "duplicate native trace identity"
+        )
+    for trace in traces:
+        if sha256(native_bytes(trace)).hexdigest() != expected.get(trace.run_id):
+            raise ImportValidationError(
+                "source_conflict", "native_trace", "trace changed after receipt capture"
+            )
+    root = Path(out)
+    root.mkdir(parents=True, exist_ok=True)
+    root = root.resolve()
+    for trace in traces:
+        write_artifact(root, f"traces/{trace.run_id}.json", native_bytes(trace))
+    for receipt in receipts:
+        write_artifact(
+            root,
+            f"receipts/{receipt.receipt_id}.json",
+            (canonical_json(receipt.to_dict()) + "\n").encode(),
+        )
+    write_artifact(root, "inventory.json", (canonical_json(inventory) + "\n").encode())
+    return root / "inventory.json"

agentloop/interoperability/cli.py

@@ -0,0 +1,59 @@
+"""Offline batch telemetry import; no collectors, SDKs or providers are started."""
+
+from __future__ import annotations
+
+from pathlib import Path
+
+import typer
+
+from agentloop.interoperability.validation import ImportValidationError
+
+telemetry_app = typer.Typer(help="Import bounded OTLP JSON/JSONL evidence offline.")
+
+
+@telemetry_app.command("import-jsonl")
+def import_jsonl_command(
+    path: Path,
+    out: Path = typer.Option(
+        ..., help="Output directory for qualified traces, receipts and inventory."
+    ),
+    source: str = typer.Option("otel", help="Source system: otel, harbor or omnigent."),
+    strict: bool = typer.Option(False, help="Fail on an independent malformed record."),
+    single_json: bool = typer.Option(
+        False, help="Read one bounded JSON document instead of JSONL."
+    ),
+    synthetic: bool = typer.Option(False, help="Explicitly mark owned synthetic fixtures."),
+) -> None:
+    """Stream a completed export and preserve every invalid/conflicting record."""
+    from agentloop.interoperability.otlp_jsonl import OtlpOptions, import_otlp
+
+    try:
+        result = import_otlp(
+            path,
+            options=OtlpOptions(
+                system=source, continue_on_error=not strict, synthetic_fixture=synthetic
+            ),
+            jsonl=not single_json,
+        )
+    except (ImportValidationError, OSError) as exc:
+        message = (
+            str(exc)
+            if isinstance(exc, ImportValidationError)
+            else "Telemetry artifacts could not be read"
+        )
+        raise typer.BadParameter(message, param_hint="path") from None
+    try:
+        inventory = result.write(out)
+    except (ImportValidationError, OSError) as exc:
+        message = (
+            str(exc) if isinstance(exc, ImportValidationError) else "Output could not be written"
+        )
+        raise typer.BadParameter(message, param_hint="--out") from None
+    stats = result.inventory()
+    typer.echo(
+        f"{stats['valid_records']} records imported; {stats['invalid_records']} invalid; {stats['native_traces']} trace projections; {stats['identical_duplicate_spans']} identical duplicate spans; {stats['conflicting_span_segments']} conflicting segments."
+    )
+    typer.echo(
+        "Source spans/evaluations are externally reported. Exported telemetry alone establishes no task-correctness gate."
+    )
+    typer.echo(f"Inventory: {inventory}")

agentloop/interoperability/cohort_evidence.py

@@ -0,0 +1,175 @@
+"""Receipt-only cohort observations with independent acceptance conditions."""
+
+from __future__ import annotations
+
+import json
+from hashlib import sha256
+from typing import Any
+
+from agentloop.integrations.harbor.trial_evidence import PAIRING_KEYS, read_trial
+from agentloop.interoperability.validation import ImportValidationError, validate_json_tree
+from agentloop.interventions import canonical_json
+
+COHORT_KEY = "agentloop.external_cohort"
+COHORT_PAIRING_KEYS = (*PAIRING_KEYS, "task_id", "verifier_config_hash")
+
+
+def _fail(reason: str) -> None:
+    raise ImportValidationError("invalid_cohort_evidence", COHORT_KEY, reason)
+
+
+def acceptance(
+    outcome: dict,
+    *,
+    import_status: str,
+    trajectory_available: bool,
+    usage_available: bool,
+    missingness_policy: str,
+) -> bool | None:
+    """Keep recorded correctness separate from usable completed-run evidence."""
+    if (
+        outcome["execution_status"] in {"failed", "timed_out", "cancelled"}
+        or outcome["quality_pass"] is False
+    ):
+        return False
+    complete = (
+        import_status == "imported"
+        and trajectory_available
+        and usage_available
+        and outcome["execution_status"] == "completed"
+        and outcome["verifier_status"] == "available"
+    )
+    if not complete or outcome["quality_pass"] is None:
+        return False if missingness_policy == "reject" else None
+    return True
+
+
+def cohort_record(
+    *,
+    source_receipt: dict | None,
+    import_status: str,
+    trajectory_available: bool,
+    usage_available: bool,
+    missingness_policy: str,
+    condition: str,
+    task_id: str | None,
+    verifier_config_hash: str | None,
+) -> dict:
+    value = {
+        "schema_version": "1.0",
+        "source_receipt_id": source_receipt["receipt_id"] if source_receipt else None,
+        "source_receipt_sha256": sha256(canonical_json(source_receipt).encode()).hexdigest()
+        if source_receipt
+        else None,
+        "import_status": import_status,
+        "trajectory_available": trajectory_available,
+        "usage_available": usage_available,
+        "missingness_policy": missingness_policy,
+        "condition": condition,
+        "task_id": task_id,
+        "verifier_config_hash": verifier_config_hash,
+    }
+    value["sha256"] = sha256(canonical_json(value).encode()).hexdigest()
+    return value
+
+
+def read_cohort(trace: Any) -> dict | None:
+    if COHORT_KEY not in trace.metadata:
+        return None
+    value = trace.metadata[COHORT_KEY]
+    validate_json_tree(value)
+    if (
+        not isinstance(value, dict)
+        or set(value)
+        != {
+            "schema_version",
+            "source_receipt_id",
+            "source_receipt_sha256",
+            "import_status",
+            "trajectory_available",
+            "usage_available",
+            "missingness_policy",
+            "condition",
+            "task_id",
+            "verifier_config_hash",
+            "sha256",
+        }
+        or value["schema_version"] != "1.0"
+    ):
+        _fail("unsupported cohort observation schema")
+    if (
+        value["sha256"]
+        != sha256(
+            canonical_json({key: item for key, item in value.items() if key != "sha256"}).encode()
+        ).hexdigest()
+    ):
+        _fail("cohort record changed after capture")
+    if (
+        type(value["trajectory_available"]) is not bool
+        or type(value["usage_available"]) is not bool
+        or not isinstance(value["missingness_policy"], str)
+        or value["missingness_policy"] not in {"indeterminate", "reject"}
+    ):
+        _fail("invalid source coverage/missingness flags")
+    if not isinstance(value["import_status"], str) or value["import_status"] not in {
+        "imported",
+        "invalid",
+        "missing",
+        "missing_trajectory",
+    }:
+        _fail("unsupported import status")
+    for key in ("condition", "task_id", "source_receipt_id"):
+        item = value[key]
+        if item is None and key != "condition":
+            continue
+        if not isinstance(item, str) or not item.strip() or len(item.encode()) > 512:
+            _fail("invalid cohort identity label")
+    for key in ("source_receipt_sha256", "verifier_config_hash"):
+        item = value[key]
+        if item is not None and (
+            not isinstance(item, str)
+            or len(item) != 64
+            or any(char not in "0123456789abcdef" for char in item)
+        ):
+            _fail("source/configuration hash must be SHA-256 or null")
+    if (value["source_receipt_id"] is None) != (value["source_receipt_sha256"] is None):
+        _fail("source receipt identity and digest must be present together")
+    if any(
+        trace.metadata.get(key) != value[key]
+        for key in ("task_id", "verifier_config_hash", "condition")
+    ):
+        _fail("source pairing labels differ from their captured record")
+    trial = read_trial(trace)
+    if trial is None:
+        _fail("cohort acceptance requires captured external trial evidence")
+    if value["source_receipt_id"] is not None and value["source_receipt_id"] != trial["receipt_id"]:
+        _fail("cohort receipt reference differs from captured trial source")
+    result = json.loads(canonical_json(value))
+    result["accepted"] = acceptance(
+        trial["outcome"],
+        import_status=value["import_status"],
+        trajectory_available=value["trajectory_available"],
+        usage_available=value["usage_available"],
+        missingness_policy=value["missingness_policy"],
+    )
+    return result
+
+
+def require_cohort_pair(baseline: Any, candidate: Any) -> None:
+    left, right = read_cohort(baseline), read_cohort(candidate)
+    if left is None and right is None:
+        return
+    if left is None or right is None:
+        _fail("a cohort comparison requires two cohort observations")
+    for key in ("task_id", "verifier_config_hash"):
+        if baseline.metadata.get(key) is None or baseline.metadata.get(
+            key
+        ) != candidate.metadata.get(key):
+            _fail("task/verifier identity is missing or incompatible")
+    if any(
+        item["import_status"] != "imported"
+        or not item["trajectory_available"]
+        or not item["usage_available"]
+        for item in (left, right)
+    ):
+        _fail("cohort source coverage is incomplete; use the full-population study report")