agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/events.py

@@ -0,0 +1,75 @@
+from __future__ import annotations
+
+from dataclasses import dataclass, field
+from datetime import datetime, timezone
+from typing import Any
+from uuid import uuid4
+
+from agentloop.operations import operation_kind
+
+
+def utc_now_iso() -> str:
+    return datetime.now(timezone.utc).isoformat()
+
+
+def new_event_id() -> str:
+    return "evt_" + uuid4().hex[:16]
+
+
+def new_run_id() -> str:
+    return "run_" + uuid4().hex[:16]
+
+
+def format_exception_detail(exc: BaseException) -> str:
+    """Return useful telemetry text even when an exception has no message."""
+
+    message = str(exc)
+    return message if message.strip() else type(exc).__name__
+
+
+@dataclass
+class AgentEvent:
+    event_id: str
+    run_id: str
+    event_type: str
+    name: str
+    started_at: str
+    ended_at: str
+    duration_ms: float
+    parent_id: str | None = None
+    model: str | None = None
+    input_tokens: int = 0
+    output_tokens: int = 0
+    # Where input_tokens/output_tokens came from. ``None`` means the trace was
+    # written before this field existed, which reads back as "unspecified" — not
+    # as exact. See agentloop.tokens and docs/TRACE_SCHEMA.md.
+    token_provenance: str | None = None
+    input_text: str | None = None
+    output_text: str | None = None
+    status: str = "ok"
+    error: str | None = None
+    metadata: dict[str, Any] = field(default_factory=dict)
+
+    @property
+    def operation_kind(self) -> str:
+        return operation_kind(self)
+
+    @property
+    def total_tokens(self) -> int:
+        return self.input_tokens + self.output_tokens
+
+    def to_dict(self) -> dict[str, Any]:
+        return self.__dict__.copy()
+
+    @classmethod
+    def from_dict(cls, data: dict[str, Any], *, index: int = 0) -> "AgentEvent":
+        """Build an event from a serialized dict, validating the schema contract.
+
+        Raises :class:`agentloop.schema.TraceValidationError` (a ``ValueError``)
+        for missing, wrongly typed, or out-of-range fields; unknown fields are
+        ignored per the documented forward-compatibility policy.
+        """
+
+        from agentloop.schema import coerce_event_dict
+
+        return cls(**coerce_event_dict(data, index=index))

agentloop/experiment_artifacts.py

@@ -0,0 +1,80 @@
+"""Owned local artifacts; exclusive journal lock and atomic immutable writes."""
+
+from __future__ import annotations
+
+import json
+import os
+from contextlib import contextmanager
+from pathlib import Path
+from uuid import uuid4
+
+from agentloop.experiment_types import canonical
+
+
+def local_path(root, relative):
+    root = Path(root).resolve()
+    target = (root / relative).resolve()
+    if root not in target.parents:
+        raise ValueError("experiment artifact path escapes its root")
+    return target
+
+
+def read_json(path):
+    return json.loads(Path(path).read_text(encoding="utf-8"))
+
+
+def write_once(root, relative, value):
+    return write_bytes_once(root, relative, (canonical(value) + "\n").encode("utf-8"))
+
+
+def write_bytes_once(root, relative, encoded):
+    path = local_path(root, relative)
+    if path.exists():
+        if path.read_bytes() != encoded:
+            raise ValueError("existing experiment artifact has conflicting content")
+        return path
+    path.parent.mkdir(parents=True, exist_ok=True)
+    temporary = local_path(
+        root, str(path.relative_to(Path(root).resolve())) + "." + uuid4().hex + ".tmp"
+    )
+    try:
+        with temporary.open("xb") as handle:
+            handle.write(encoded)
+            handle.flush()
+            os.fsync(handle.fileno())
+        # All journal writers hold the same exclusive lock. Existing artifacts
+        # are compared above; an interrupted temporary is never a completed record.
+        os.replace(temporary, path)
+    finally:
+        temporary.unlink(missing_ok=True)
+    return path
+
+
+@contextmanager
+def journal_lock(root):
+    root = Path(root).resolve()
+    root.mkdir(parents=True, exist_ok=True)
+    path = local_path(root, "journal.lock")
+    nonce = uuid4().hex
+    try:
+        with path.open("x", encoding="utf-8") as handle:
+            handle.write(nonce)
+            handle.flush()
+            os.fsync(handle.fileno())
+    except FileExistsError:
+        raise ValueError(
+            "experiment journal is busy or requires host lock reconciliation"
+        ) from None
+    failed = False
+    try:
+        yield
+    except BaseException:
+        failed = True
+        raise
+    finally:
+        try:
+            if path.read_text(encoding="utf-8") == nonce:
+                path.unlink()
+        except BaseException:
+            if not failed:
+                raise

agentloop/experiment_observations.py

@@ -0,0 +1,169 @@
+"""Bounded scalar observations attached to native experiment traces."""
+
+from __future__ import annotations
+
+import math
+from collections import Counter
+
+from agentloop.context_types import identifier
+from agentloop.experiment_types import owned, reference
+from agentloop.studies import summarize_values
+from agentloop.tracer import current_trace
+
+OBSERVATIONS_KEY = "agentloop.experiment_observations"
+
+
+def observation(value, *, kind="observed", unit="count", source_ref):
+    if value is not None and not (
+        type(value) in {bool, int, float, str} and (not isinstance(value, str) or len(value) <= 256)
+    ):
+        raise ValueError("experiment observations must be bounded scalars")
+    if type(value) in {int, float}:
+        try:
+            if not math.isfinite(value):
+                raise ValueError("experiment observations must be finite")
+        except OverflowError:
+            raise ValueError("experiment observation is outside the numeric range") from None
+    if kind not in {"observed", "estimated", "declared"}:
+        raise ValueError("unknown observation kind")
+    identifier(unit, "observation unit")
+    reference(source_ref, "observation source")
+    return {"value": value, "kind": kind, "unit": unit, "source_ref": source_ref}
+
+
+def record_experiment_observations(values):
+    trace = current_trace()
+    if trace is None or not isinstance(trace.metadata.get("experiment_id"), str):
+        raise ValueError("observations require an active experiment trace")
+    if not isinstance(values, dict) or len(values) > 64:
+        raise ValueError("observations require a bounded mapping")
+    copied = {}
+    for key, value in values.items():
+        identifier(key, "observation name")
+        if not isinstance(value, dict) or set(value) != {"value", "kind", "unit", "source_ref"}:
+            raise ValueError("invalid observation fields")
+        copied[key] = observation(**value)
+    envelope = trace.metadata.setdefault(OBSERVATIONS_KEY, {"schema_version": "1.0", "values": {}})
+    if (
+        not isinstance(envelope, dict)
+        or envelope.get("schema_version") != "1.0"
+        or not isinstance(envelope.get("values"), dict)
+    ):
+        raise ValueError("invalid experiment observation envelope")
+    if len(set(envelope["values"]) | set(copied)) > 64:
+        raise ValueError("experiment observation limit exceeded")
+    if any(
+        key in envelope["values"] and envelope["values"][key] != value
+        for key, value in copied.items()
+    ):
+        raise ValueError("experiment observation cannot be rewritten")
+    envelope["values"].update(owned(copied))
+
+
+def summarize_observations(rows):
+    sets = []
+    for row in rows:
+        raw = (row["trace"] or {}).get("metadata", {}).get(OBSERVATIONS_KEY)
+        if raw is None:
+            sets.append({})
+            continue
+        if (
+            not isinstance(raw, dict)
+            or raw.get("schema_version") != "1.0"
+            or not isinstance(raw.get("values"), dict)
+            or len(raw["values"]) > 64
+        ):
+            raise ValueError("invalid saved experiment observations")
+        values = {}
+        for name, item in raw["values"].items():
+            identifier(name, "observation name")
+            values[name] = observation(**item)
+        sets.append(values)
+    result = {}
+    for name in sorted({key for values in sets for key in values}):
+        available = [
+            values[name] for values in sets if name in values and values[name]["value"] is not None
+        ]
+        kinds = sorted({item["kind"] for item in available})
+        units = sorted({item["unit"] for item in available})
+        sources = sorted({item["source_ref"] for item in available})
+        row_values = [values.get(name, {}).get("value") for values in sets]
+        compatible = len(kinds) == len(units) == len(sources) == 1
+        summary = {
+            "planned_count": len(rows),
+            "known_count": len(available),
+            "missing_count": len(rows) - len(available),
+            "kinds": kinds,
+            "units": units,
+            "source_refs": sources,
+            "aggregation_status": "compatible" if compatible else "unknown_or_mixed_basis",
+        }
+        if compatible and all(type(value) is bool for value in row_values if value is not None):
+            true_count = sum(value is True for value in row_values)
+            quality_true = sum(
+                value is True and row["receipt"] is not None and row["receipt"]["quality_passed"]
+                for value, row in zip(row_values, rows)
+            )
+            summary.update(
+                true_count=true_count,
+                false_count=sum(value is False for value in row_values),
+                true_fraction_of_planned=true_count / len(rows),
+                quality_preserving_true_count=quality_true,
+                quality_preserving_true_fraction_of_planned=quality_true / len(rows),
+            )
+        elif compatible and all(
+            type(value) in {int, float} for value in row_values if value is not None
+        ):
+            summary["values"] = summarize_values(row_values)
+            summary["quality_preserving_values"] = summarize_values(
+                [
+                    value
+                    if row["receipt"] is not None and row["receipt"]["quality_passed"]
+                    else None
+                    for value, row in zip(row_values, rows)
+                ]
+            )
+            if units == ["completed_items"] and kinds == ["observed"]:
+                rates = []
+                for value, row in zip(row_values, rows):
+                    elapsed = (
+                        None
+                        if row["intervention"] is None
+                        else row["intervention"]["measured"]["candidate"]["runtime_ms"]
+                    )
+                    rate = (
+                        value * 1000 / elapsed
+                        if value is not None and elapsed is not None and elapsed > 0
+                        else None
+                    )
+                    rates.append(rate if rate is not None and math.isfinite(rate) else None)
+                summary["per_second_of_end_to_end_candidate_runtime"] = summarize_values(rates)
+                summary["quality_preserving_per_second"] = summarize_values(
+                    [
+                        value
+                        if row["receipt"] is not None and row["receipt"]["quality_passed"]
+                        else None
+                        for value, row in zip(rates, rows)
+                    ]
+                )
+        elif compatible:
+            summary["value_counts"] = dict(
+                sorted(Counter(str(value) for value in row_values if value is not None).items())
+            )
+            summary["quality_pass_counts_by_value"] = dict(
+                sorted(
+                    Counter(
+                        str(value)
+                        for value, row in zip(row_values, rows)
+                        if value is not None
+                        and row["receipt"] is not None
+                        and row["receipt"]["quality_passed"]
+                    ).items()
+                )
+            )
+            summary["quality_fraction_by_value"] = {
+                key: summary["quality_pass_counts_by_value"].get(key, 0) / count
+                for key, count in summary["value_counts"].items()
+            }
+        result[name] = summary
+    return result