agentloop-profiler 0.1.0__py3-none-any.whl

agentloop/substitutions.py

@@ -0,0 +1,267 @@
+"""Explicit offline candidate comparisons and native paired-study artifacts."""
+
+from __future__ import annotations
+
+import inspect
+import json
+import platform
+import random
+from time import perf_counter
+
+from agentloop.events import utc_now_iso
+from agentloop.judgment_types import JudgeUsage, canonical, fingerprint
+from agentloop.quality import build_quality_report
+from agentloop.substitution_evidence import attach_trial_evidence, read_trial_evidence
+from agentloop.substitution_types import (
+    SUBSTITUTION_VERSION,
+    DecisionImplementation,
+    DecisionRequest,
+    DecisionResult,
+    SubstitutionProtocol,
+)
+from agentloop.tracer import AgentTrace, bind_trace_context
+from agentloop.workflows import StageInfo, WorkflowInfo, record_operation, workflow_metadata
+
+
+def _copy(value):
+    return json.loads(canonical(value))
+
+
+def _trace(
+    protocol, example, repeat, declaration, plan_hash, outcome, started, ended, latency, output_hash
+):
+    failed = outcome in {"failed", "timed_out", "invalid_result"}
+    status = (
+        "failed" if failed else ("unknown" if outcome in {"disabled", "missing"} else "completed")
+    )
+    metadata = workflow_metadata(
+        WorkflowInfo("offline-model-substitution", protocol.version),
+        task_id=example.example_id,
+        status=status,
+        metadata={
+            "synthetic": protocol.synthetic,
+            "source": "agentloop_substitution",
+            "plan_hash": plan_hash,
+            "example_id": example.example_id,
+            "repetition": repeat,
+            "condition": declaration["name"],
+        },
+    )
+    trace = AgentTrace(
+        "Decision trial: " + declaration["name"],
+        metadata=metadata,
+        started_at=started,
+        ended_at=ended,
+        elapsed_ms=latency or 0,
+    )
+    if latency is not None:
+        record_operation(
+            protocol.step.step_id,
+            kind="classifier",
+            duration_ms=latency,
+            started_at=started,
+            ended_at=ended,
+            stage=StageInfo(
+                protocol.step.step_id,
+                declaration["identity"]["version"],
+                kind=declaration["kind"],
+                input_ref=example.input_ref,
+            ),
+            outcome=outcome,
+            output_ref="sha256:" + output_hash if output_hash is not None else None,
+            status="error" if failed else "ok",
+            metadata={"error_type": outcome} if failed else None,
+            trace=trace,
+            event_id="decision",
+        )
+    return trace
+
+
+def _execute(protocol, example, repeat, implementation, declaration, plan_hash, enabled):
+    started = utc_now_iso()
+    status, usage, cost_ref, output, output_hash = (
+        "disabled",
+        JudgeUsage(0, 0, 0, "not_dispatched", "not_dispatched"),
+        None,
+        None,
+        None,
+    )
+    latency = None
+    if enabled:
+        usage = JudgeUsage()
+        request = DecisionRequest(
+            example.example_id, example.input_ref, protocol.step, example._input_json
+        )
+        request.inputs  # Decode the isolated input before measuring the callback.
+        started = utc_now_iso()
+        began = perf_counter()
+        try:
+            # Experimental callbacks cannot append work to an ambient live run.
+            with bind_trace_context(None):
+                value = implementation.invoke(request, timeout_s=protocol.timeout_s)
+            latency = (perf_counter() - began) * 1000
+            ended = utc_now_iso()
+            if type(value) is not DecisionResult:
+                if inspect.iscoroutine(value) or inspect.isgenerator(value):
+                    value.close()
+                status = "invalid_result"
+            else:
+                status, usage, cost_ref = value.status, value.usage, value.cost_ref
+                if status == "completed":
+                    output = value.output
+                    output_hash = fingerprint(output)
+            if implementation.declaration() != declaration:
+                status, output, output_hash = "invalid_result", None, None
+        except TimeoutError:
+            status = "timed_out"
+        except Exception:
+            status = "failed"
+        if latency is None:
+            latency = (perf_counter() - began) * 1000
+            ended = utc_now_iso()
+        if protocol.timeout_s is not None and latency > protocol.timeout_s * 1000:
+            status, output, output_hash = "timed_out", None, None
+    else:
+        ended = started
+    trace = _trace(
+        protocol,
+        example,
+        repeat,
+        declaration,
+        plan_hash,
+        status,
+        started,
+        ended,
+        latency,
+        output_hash,
+    )
+    attach_trial_evidence(
+        trace,
+        plan_hash=plan_hash,
+        example_id=example.example_id,
+        repetition=repeat,
+        implementation=declaration,
+        status=status,
+        latency_ms=latency,
+        usage=usage,
+        cost_ref=cost_ref,
+        input_hash=fingerprint(example.inputs),
+        output_hash=output_hash,
+    )
+    return trace, output
+
+
+def run_substitution_experiment(
+    protocol, baseline, candidates, *, enabled=False, max_invocations=10000
+):
+    """Run declared implementations, then freeze quality assessments for export.
+
+    Both baseline and candidates execute when enabled. Reference outputs and
+    scorer configuration are never passed to an implementation. The host owns
+    callback/scorer side effects, budgets and cooperative cancellation.
+    """
+    if (
+        type(protocol) is not SubstitutionProtocol
+        or type(baseline) is not DecisionImplementation
+        or type(enabled) is not bool
+    ):
+        raise ValueError("experiment requires a typed protocol, baseline and boolean enabled")
+    if (
+        not isinstance(candidates, (tuple, list))
+        or not candidates
+        or any(type(item) is not DecisionImplementation for item in candidates)
+    ):
+        raise ValueError("at least one explicit candidate is required")
+    implementations = [baseline, *candidates]
+    by_name = {item.name: item for item in implementations}
+    if len(by_name) != len(implementations):
+        raise ValueError("baseline and candidate names must be unique")
+    count = len(implementations) * len(protocol.examples) * protocol.repetitions
+    if type(max_invocations) is not int or max_invocations < 1 or count > max_invocations:
+        raise ValueError("planned experiment exceeds max_invocations")
+    declarations = {item.name: item.declaration() for item in implementations}
+    plan = {
+        "protocol": protocol.declaration(),
+        "baseline": baseline.name,
+        "implementations": declarations,
+    }
+    plan_hash = fingerprint(plan)
+    jobs = [
+        (name, index, repeat)
+        for name in sorted(by_name)
+        for index in range(len(protocol.examples))
+        for repeat in range(protocol.repetitions)
+    ]
+    random.Random(protocol.seed).shuffle(jobs)  # nosec B311
+    rows, completed = [], {}
+    for ordinal, (name, index, repeat) in enumerate(jobs):
+        example = protocol.examples[index]
+        trace, output = _execute(
+            protocol, example, repeat, by_name[name], declarations[name], plan_hash, enabled
+        )
+        rows.append(
+            {
+                "ordinal": ordinal,
+                "condition": name,
+                "example_id": example.example_id,
+                "repetition": repeat,
+                "trace": trace.to_dict(),
+            }
+        )
+        completed[name, example.example_id, repeat] = trace, output
+    pairs = []
+    for candidate in candidates:
+        for example in protocol.examples:
+            for repeat in range(protocol.repetitions):
+                before, old_output = completed[baseline.name, example.example_id, repeat]
+                after, new_output = completed[candidate.name, example.example_id, repeat]
+                before_receipt, after_receipt = (
+                    read_trial_evidence(before),
+                    read_trial_evidence(after),
+                )
+                fixture = example.fixture(protocol.scorer)
+                if before_receipt["outcome"] == "completed":
+                    fixture["baseline_output"] = old_output
+                if after_receipt["outcome"] == "completed":
+                    fixture["candidate_output"] = new_output
+                quality = None
+                quality_error = None
+                try:
+                    quality = build_quality_report(
+                        [fixture],
+                        baseline_trace=before,
+                        candidate_trace=after,
+                        min_score=protocol.min_quality,
+                    )
+                except Exception:
+                    quality_error = "assessment_failed"
+                agreement = None
+                if before_receipt["outcome"] == after_receipt["outcome"] == "completed":
+                    agreement = canonical(old_output) == canonical(new_output)
+                pairs.append(
+                    {
+                        "condition": candidate.name,
+                        "example_id": example.example_id,
+                        "repetition": repeat,
+                        "baseline_run_id": before.run_id,
+                        "candidate_run_id": after.run_id,
+                        "quality": quality,
+                        "quality_error": quality_error,
+                        "baseline_agreement": agreement,
+                    }
+                )
+    return _copy(
+        {
+            "schema_version": SUBSTITUTION_VERSION,
+            **plan,
+            "plan_hash": plan_hash,
+            "enabled": enabled,
+            "records": rows,
+            "pairs": pairs,
+            "environment": {
+                "python": platform.python_version(),
+                "system": platform.system(),
+                "architecture": platform.machine(),
+            },
+        }
+    )

agentloop/timing.py

@@ -0,0 +1,142 @@
+from __future__ import annotations
+
+import math
+import time
+from datetime import datetime, timezone
+from typing import Any, Iterable
+
+
+def format_duration_ms(value: float | None, *, unit="s", precision=2) -> str:
+    """Render an optional timing estimate without turning unavailable into zero."""
+    if value is None:
+        return "unavailable"
+    if unit not in {"s", "ms"}:
+        raise ValueError("unit must be s or ms")
+    amount = value / 1000 if unit == "s" else value
+    suffix = "s" if unit == "s" else " ms"
+    return f"{amount:.{precision}f}{suffix}"
+
+
+def timestamp_ms(value: Any) -> float | None:
+    """Parse an ISO-8601 timestamp into milliseconds since the Unix epoch."""
+
+    if not isinstance(value, str) or not value:
+        return None
+    try:
+        parsed = datetime.fromisoformat(value.replace("Z", "+00:00"))
+    except ValueError:
+        return None
+    if parsed.tzinfo is None:
+        parsed = parsed.replace(tzinfo=timezone.utc)
+    result = parsed.timestamp() * 1000
+    return result if math.isfinite(result) else None
+
+
+def duration_ms(value: Any) -> float:
+    """Return a finite, non-negative duration for an event-like object."""
+
+    try:
+        result = float(getattr(value, "duration_ms", 0.0) or 0.0)
+    except (TypeError, ValueError):
+        return 0.0
+    return result if math.isfinite(result) and result >= 0 else 0.0
+
+
+def event_interval_ms(event: Any) -> tuple[float, float] | None:
+    """Return a trustworthy timestamp interval for an event, when available.
+
+    Older adapters sometimes filled both timestamps with the current time while
+    carrying the real duration in ``duration_ms``. Treat a large disagreement
+    between those values as missing timing data so legacy traces can use the
+    cumulative-duration fallback instead of reporting an almost-zero runtime.
+    """
+
+    metadata = getattr(event, "metadata", {})
+    if (
+        isinstance(metadata, dict)
+        and metadata.get("external_evidence_schema") == "1.0"
+        and metadata.get("timing_available") is False
+    ):
+        return None
+    started = timestamp_ms(getattr(event, "started_at", None))
+    ended = timestamp_ms(getattr(event, "ended_at", None))
+    if started is None or ended is None or ended < started:
+        return None
+
+    recorded_duration = duration_ms(event)
+    observed_duration = ended - started
+    if recorded_duration > 0:
+        tolerance = max(5.0, recorded_duration * 0.10)
+        if abs(observed_duration - recorded_duration) > tolerance:
+            return None
+    return started, ended
+
+
+def event_time_bounds(events: Iterable[Any]) -> tuple[float, float] | None:
+    """Return the earliest reliable start and latest reliable end.
+
+    A positive-duration event without trustworthy timestamps makes the complete
+    trace envelope unknowable, so callers should fall back to cumulative work.
+    Zero-duration events without timestamps do not affect either result.
+    """
+
+    earliest: float | None = None
+    latest: float | None = None
+    for event in events:
+        interval = event_interval_ms(event)
+        if interval is None:
+            if duration_ms(event) > 0:
+                return None
+            continue
+        started, ended = interval
+        earliest = started if earliest is None else min(earliest, started)
+        latest = ended if latest is None else max(latest, ended)
+    if earliest is None or latest is None:
+        return None
+    return earliest, latest
+
+
+def cumulative_span_time_ms(events: Iterable[Any]) -> float:
+    """Return total instrumented work, including overlap and nesting."""
+
+    return sum(duration_ms(event) for event in events)
+
+
+def elapsed_runtime_ms(trace: Any) -> float:
+    """Resolve end-to-end trace duration with backward-compatible fallbacks."""
+
+    explicit = _non_negative_float(getattr(trace, "elapsed_ms", None))
+    if explicit is not None:
+        return explicit
+
+    if getattr(trace, "_timing_active", False):
+        started_perf = _non_negative_float(getattr(trace, "_start_perf", None))
+        if started_perf is not None:
+            return max(0.0, (time.perf_counter() - started_perf) * 1000)
+
+    trace_started = timestamp_ms(getattr(trace, "started_at", None))
+    trace_ended = timestamp_ms(getattr(trace, "ended_at", None))
+    if trace_started is not None and trace_ended is not None and trace_ended >= trace_started:
+        return trace_ended - trace_started
+
+    events = list(getattr(trace, "events", []))
+    bounds = event_time_bounds(events)
+    if bounds is not None:
+        earliest, latest = bounds
+        if trace_started is not None and trace_started <= latest:
+            earliest = min(earliest, trace_started)
+        return max(0.0, latest - earliest)
+
+    return cumulative_span_time_ms(events)
+
+
+def _non_negative_float(value: Any) -> float | None:
+    if value is None or value == "":
+        return None
+    try:
+        result = float(value)
+    except (TypeError, ValueError):
+        return None
+    if not math.isfinite(result) or result < 0:
+        return None
+    return result

agentloop/tokens.py

@@ -0,0 +1,188 @@
+"""Token-count provenance: where a token number came from, and how exact it is.
+
+The core rule of this module mirrors :mod:`agentloop.costs`: **a measurement is
+only ever presented as exact when it can be justified.** AgentLoop stores a
+whitespace word-count fallback in the same ``input_tokens`` / ``output_tokens``
+fields that carry provider-reported usage, so without provenance a trace can show
+an exact-looking token count — and an exact-looking dollar amount derived from it
+— that actually came from ``len(text.split())``.
+
+Seven provenance values are represented (`TokenProvenance`), recorded per event:
+
+- ``provider``       — the provider reported usage (e.g. an OpenAI ``usage`` object).
+- ``tokenizer``      — counted with a real tokenizer for the target model.
+- ``user_supplied``  — explicit counts passed by the calling application.
+- ``external_reported`` — imported counts without verified provider accounting.
+- ``estimated_words`` — the whitespace word-count fallback; an approximation.
+- ``unavailable``    — no counts were available and none could be estimated.
+- ``unspecified``    — the trace predates this field, so provenance is unknown.
+
+The first three are *exact-grade*: they are counts of tokens rather than a proxy
+for them. ``estimated_words`` is explicitly an approximation. Aggregating those
+grades over a trace's model calls yields a `TokenStatus`, which is what reports,
+cost breakdowns, and replay gates consult. See ``docs/TRACE_SCHEMA.md`` for the
+serialized contract and ``docs/RESEARCH.md`` for which measurements are suitable
+to report as experimental results.
+"""
+
+from __future__ import annotations
+
+from typing import Any, Literal
+
+TokenProvenance = Literal[
+    "provider",
+    "tokenizer",
+    "user_supplied",
+    "external_reported",
+    "estimated_words",
+    "unavailable",
+    "unspecified",
+]
+TokenStatus = Literal[
+    "exact", "estimated", "partial", "unavailable", "unspecified", "empty", "external_reported"
+]
+
+PROVIDER = "provider"
+TOKENIZER = "tokenizer"
+USER_SUPPLIED = "user_supplied"
+EXTERNAL_REPORTED = "external_reported"
+ESTIMATED_WORDS = "estimated_words"
+UNAVAILABLE = "unavailable"
+UNSPECIFIED = "unspecified"
+
+#: Provenance values a producer may record. ``unspecified`` is a *read* result
+#: for traces written before the field existed, never something to write.
+WRITABLE_PROVENANCE = frozenset(
+    {PROVIDER, TOKENIZER, USER_SUPPLIED, EXTERNAL_REPORTED, ESTIMATED_WORDS, UNAVAILABLE}
+)
+
+#: Provenance values that count tokens rather than approximate them.
+EXACT_PROVENANCE = frozenset({PROVIDER, TOKENIZER, USER_SUPPLIED})
+
+# Exactness grades, in the order the aggregate status resolves them.
+_EXACT = "exact"
+_ESTIMATED = "estimated"
+_UNAVAILABLE = "unavailable"
+_UNSPECIFIED = "unspecified"
+
+
+class TokenProvenanceError(ValueError):
+    """A producer tried to record a token provenance AgentLoop does not define."""
+
+
+def validate_provenance(value: Any) -> str:
+    """Return ``value`` if a producer may record it, else raise.
+
+    Producers (the tracer and the integrations) validate strictly so a typo fails
+    fast at the point it is introduced. The *deserialization* path deliberately
+    does not: see :func:`provenance_grade` for why unrecognized values read back
+    as ``unspecified`` instead of rejecting a trace from a newer minor version.
+    """
+
+    if value not in WRITABLE_PROVENANCE:
+        raise TokenProvenanceError(
+            f"unknown token provenance {value!r}; expected one of "
+            f"{', '.join(sorted(WRITABLE_PROVENANCE))}"
+        )
+    return str(value)
+
+
+def provenance_grade(value: Any) -> str:
+    """Return the exactness grade for one event's provenance value.
+
+    A missing value (an event from a trace written before this field existed) and
+    any value this build does not recognize both grade as ``unspecified``. Being
+    lenient here is what keeps the schema's MINOR forward-compatibility promise:
+    a trace written by a newer AgentLoop that adds a provenance value stays
+    readable, and its unknown value is simply not claimed to be exact.
+    """
+
+    if value is None:
+        return _UNSPECIFIED
+    if value in EXACT_PROVENANCE:
+        return _EXACT
+    if value == ESTIMATED_WORDS:
+        return _ESTIMATED
+    if value == UNAVAILABLE:
+        return _UNAVAILABLE
+    if value == EXTERNAL_REPORTED:
+        return EXTERNAL_REPORTED
+    return _UNSPECIFIED
+
+
+def token_status(model_events: list[Any]) -> str:
+    """Summarize how exact a trace's token totals are.
+
+    - ``empty``       — no model calls, so there is no token count to qualify.
+    - ``exact``       — every model call counted tokens (provider/tokenizer/user).
+    - ``partial``     — some model calls counted tokens and some did not.
+    - ``estimated``   — no exact counts, and at least one is a word approximation.
+    - ``unavailable`` — every model call recorded that it had no counts at all.
+    - ``unspecified`` — provenance is unknown for every model call (legacy trace).
+
+    Consumers should treat anything other than ``exact``/``empty`` as "do not
+    present these token totals, or a cost derived from them, as an exact
+    measurement."
+    """
+
+    grades = {provenance_grade(getattr(event, "token_provenance", None)) for event in model_events}
+    if not grades:
+        return "empty"
+    if grades == {_EXACT}:
+        return "exact"
+    if _EXACT in grades:
+        return "partial"
+    if _ESTIMATED in grades:
+        return "estimated"
+    if grades == {_UNAVAILABLE}:
+        return "unavailable"
+    if grades == {EXTERNAL_REPORTED}:
+        return EXTERNAL_REPORTED
+    if EXTERNAL_REPORTED in grades:
+        return "partial"
+    return "unspecified"
+
+
+def provenance_counts(model_events: list[Any]) -> dict[str, int]:
+    """Count model calls per recorded provenance value, for report transparency."""
+
+    counts: dict[str, int] = {}
+    for event in model_events:
+        value = getattr(event, "token_provenance", None)
+        key = str(value) if value is not None else UNSPECIFIED
+        counts[key] = counts.get(key, 0) + 1
+    return dict(sorted(counts.items()))
+
+
+def is_token_basis_exact(status: str | None) -> bool:
+    """Return whether token totals are exact enough to report as measurements."""
+
+    return status in {"exact", "empty"}
+
+
+def is_token_basis_evaluable(status: str | None) -> bool:
+    """Return whether token totals may back a cost comparison or gate.
+
+    Stricter than nothing, looser than :func:`is_token_basis_exact`: a trace with
+    *unknown* provenance (``unspecified``) still gates, because every trace
+    written before this field existed would otherwise turn every configured cost
+    gate indeterminate on upgrade. A trace that positively reports a word
+    estimate does not gate — there the producer told us the number is a proxy.
+    See ``docs/TRACE_SCHEMA.md`` for the compatibility path.
+    """
+
+    return status in {"exact", "empty", "unspecified", None}
+
+
+def describe_token_status(status: str | None) -> str:
+    """Return a short human explanation of a token status, for reports and CLI."""
+
+    return {
+        "external_reported": "external counts; provider accounting is not independently verified",
+        "exact": "counted tokens (provider, tokenizer, or caller-supplied)",
+        "partial": "mixed: some model calls counted tokens, some did not",
+        "estimated": "approximated from whitespace word counts, not counted tokens",
+        "unavailable": "no token counts were available",
+        "unspecified": "unknown: trace predates token provenance",
+        "empty": "no model calls",
+    }.get(str(status), "unknown token basis")