agentloop-profiler 0.4.0__py3-none-any.whl

agentloop/integrations/vercel_ai.py

@@ -0,0 +1,83 @@
+from __future__ import annotations
+
+from typing import Any
+
+from agentloop.events import AgentEvent, new_event_id, utc_now_iso
+from agentloop.tracer import AgentTrace
+
+
+def _get(data: Any, key: str, default: Any = None) -> Any:
+    if isinstance(data, dict):
+        return data.get(key, default)
+    return getattr(data, key, default)
+
+
+def trace_from_vercel_ai_events(
+    events: list[dict[str, Any]],
+    *,
+    name: str = "vercel_ai_run",
+    run_id: str | None = None,
+    metadata: dict[str, Any] | None = None,
+) -> AgentTrace:
+    """Convert Vercel AI SDK-style telemetry events into an AgentLoop trace.
+
+    The adapter intentionally accepts plain dictionaries so JS/TS apps can POST or
+    export telemetry without importing Python code. Expected fields are flexible:
+    `type`/`event_type`, `name`, `duration_ms`, `model`, `input_tokens`,
+    `output_tokens`, `status`, and `metadata`.
+    """
+
+    trace = AgentTrace(
+        name=name, run_id=run_id, metadata={"integration": "vercel_ai", **(metadata or {})}
+    )
+    for item in events:
+        event_type = str(_get(item, "event_type", _get(item, "type", "model_call")))
+        if event_type in {"text", "generate", "stream", "model"}:
+            event_type = "model_call"
+        elif event_type in {"tool", "tool-call", "step"}:
+            event_type = "tool_call"
+
+        trace.add_event(
+            AgentEvent(
+                event_id=str(_get(item, "event_id", new_event_id())),
+                run_id=trace.run_id,
+                event_type=event_type,
+                name=str(_get(item, "name", event_type)),
+                started_at=str(_get(item, "started_at", utc_now_iso())),
+                ended_at=str(_get(item, "ended_at", utc_now_iso())),
+                duration_ms=float(_get(item, "duration_ms", 0.0) or 0.0),
+                parent_id=_get(item, "parent_id"),
+                model=_get(item, "model"),
+                input_tokens=int(_get(item, "input_tokens", _get(item, "prompt_tokens", 0)) or 0),
+                output_tokens=int(
+                    _get(item, "output_tokens", _get(item, "completion_tokens", 0)) or 0
+                ),
+                input_text=_get(item, "input_text"),
+                output_text=_get(item, "output_text"),
+                status=str(_get(item, "status", "ok")),
+                error=_get(item, "error"),
+                metadata={"integration": "vercel_ai", **(_get(item, "metadata", {}) or {})},
+            )
+        )
+    return trace
+
+
+TYPESCRIPT_SNIPPET = r"""
+// Minimal Vercel AI SDK telemetry bridge for AgentLoop.
+// Push these events to your backend, then convert them with
+// agentloop.integrations.vercel_ai.trace_from_vercel_ai_events(...).
+
+const agentloopEvents: any[] = [];
+
+export function recordAgentLoopEvent(event: any) {
+  agentloopEvents.push({
+    ...event,
+    started_at: event.started_at ?? new Date().toISOString(),
+    ended_at: event.ended_at ?? new Date().toISOString(),
+  });
+}
+
+export function getAgentLoopEvents() {
+  return agentloopEvents;
+}
+"""

agentloop/issues.py

@@ -0,0 +1,88 @@
+from __future__ import annotations
+
+from typing import Any
+
+from agentloop.markdown import markdown_code_span, markdown_heading, markdown_text
+
+# Generated drafts target this repository, so use only deliberately maintained
+# labels that already exist. Finding type and severity remain structured fields
+# in the draft body instead of creating an unbounded dynamic label taxonomy.
+ISSUE_DRAFT_LABELS = ("enhancement",)
+
+
+def build_issue_drafts(queue: list[dict[str, Any]], *, limit: int = 5) -> list[dict[str, Any]]:
+    drafts = []
+    for item in queue:
+        if item.get("patchable_count", 0) <= 0:
+            continue
+        drafts.append(_issue_for_queue_item(item))
+        if len(drafts) >= limit:
+            break
+    return drafts
+
+
+def issue_drafts_to_markdown(drafts: list[dict[str, Any]]) -> str:
+    lines = ["# AgentLoop GitHub Issue Drafts", ""]
+    if not drafts:
+        lines.append("No patchable optimization queue items found.")
+        return "\n".join(lines).rstrip() + "\n"
+
+    for draft in drafts:
+        lines.extend(
+            [
+                f"## {markdown_heading(draft['title'])}",
+                "",
+                f"- Labels: {markdown_text(', '.join(map(str, draft['labels'])))}",
+                "",
+                draft["body"],
+                "",
+            ]
+        )
+    return "\n".join(lines).rstrip() + "\n"
+
+
+def _issue_for_queue_item(item: dict[str, Any]) -> dict[str, Any]:
+    title = f"Optimize agent workflow: {item['title']}"
+    labels = list(ISSUE_DRAFT_LABELS)
+    body = "\n".join(
+        [
+            "AgentLoop detected a recurring patchable optimization opportunity.",
+            "",
+            "## Finding",
+            "",
+            f"- Type: {markdown_code_span(item['type'])}",
+            f"- Severity: {markdown_code_span(item['severity'])}",
+            f"- Occurrences: {item['occurrence_count']}",
+            f"- Affected runs: {item['run_count']}",
+            f"- Patchable findings: {item['patchable_count']}",
+            f"- Quality risk: {markdown_code_span(item.get('quality_risk', 'unknown'))}",
+            f"- Requires scorer: {item.get('requires_scorer', True)}",
+            f"- Safe to auto-patch: {item.get('safe_to_auto_patch', False)}",
+            f"- Estimated latency savings: {item['estimated_latency_savings_ms'] / 1000:.2f}s",
+            f"- Estimated cost savings: ${item['estimated_cost_savings_usd']:.4f}",
+            f"- Priority score: {item['priority_score']:.1f}",
+            "",
+            "## Affected Runs",
+            "",
+            "\n".join(
+                f"- {markdown_code_span(run_id)}" for run_id in item.get("affected_runs", [])[:20]
+            ),
+            "",
+            "## Acceptance Criteria",
+            "",
+            "- Generate or inspect the AgentLoop patch plan for one affected run.",
+            "- Apply a constrained workflow rewrite.",
+            "- Run `agentloop replay` against baseline and candidate traces.",
+            "- Attach a quality fixture or scorer when quality risk is medium or high.",
+            "- Run `agentloop ci` and confirm cost, latency, retry, schema, and quality gates pass.",
+        ]
+    )
+    return {
+        "title": title,
+        "body": body,
+        "labels": labels,
+        "queue_id": item["queue_id"],
+        "type": item["type"],
+        "severity": item["severity"],
+        "priority_score": item["priority_score"],
+    }

agentloop/markdown.py

@@ -0,0 +1,102 @@
+from __future__ import annotations
+
+import re
+from typing import Any
+
+_LINE_BREAKS = re.compile(r"\r\n?|\n|\x85|\u2028|\u2029")
+_BACKTICK_RUNS = re.compile(r"`+")
+_SAFE_INFO_STRING = re.compile(r"[A-Za-z0-9_.+-]*\Z")
+
+# Markdown treats these characters as inline syntax, while HTML parsers give
+# angle brackets and ampersands additional meaning. Character references keep
+# the rendered text unchanged without relying on escape ordering.
+_INLINE_ESCAPES = {
+    "&": "&",
+    "<": "&lt;",
+    ">": "&gt;",
+    "\\": "&#92;",
+    "`": "&#96;",
+    "*": "&#42;",
+    "_": "&#95;",
+    "[": "&#91;",
+    "]": "&#93;",
+    "~": "&#126;",
+}
+_TABLE_TRANSLATION = str.maketrans({"|": "&#124;"})
+
+
+def markdown_text(value: Any) -> str:
+    """Render an untrusted value as single-line Markdown text."""
+
+    content = _single_line(value)
+    return "".join(
+        character
+        if character == "_" and _underscore_is_inside_identifier(content, index)
+        else _INLINE_ESCAPES.get(character, character)
+        for index, character in enumerate(content)
+    )
+
+
+def markdown_heading(value: Any) -> str:
+    """Render an untrusted value inside a heading without allowing new blocks."""
+
+    return markdown_text(value)
+
+
+def markdown_table_cell(value: Any) -> str:
+    """Render an untrusted value without adding Markdown table columns or rows."""
+
+    return markdown_text(value).translate(_TABLE_TRANSLATION)
+
+
+def markdown_code_span(value: Any) -> str:
+    """Render an untrusted value in an inline code span with a safe delimiter."""
+
+    content = _single_line(value)
+    if not content:
+        return ""
+
+    delimiter = "`" * (_longest_backtick_run(content) + 1)
+    needs_padding = content.startswith("`") or content.endswith("`")
+    if content.strip(" ") and (content.startswith(" ") or content.endswith(" ")):
+        needs_padding = True
+    padding = " " if needs_padding else ""
+    return f"{delimiter}{padding}{content}{padding}{delimiter}"
+
+
+def markdown_fenced_code(value: Any, *, language: str = "") -> str:
+    """Render untrusted multiline code using a fence longer than its contents."""
+
+    if not _SAFE_INFO_STRING.fullmatch(language):
+        raise ValueError("Markdown fence language contains unsafe characters")
+
+    content = _normalize_line_breaks(_string(value))
+    fence = "`" * max(3, _longest_backtick_run(content) + 1)
+    opening = f"{fence}{language}"
+    separator = "" if content.endswith("\n") else "\n"
+    return f"{opening}\n{content}{separator}{fence}"
+
+
+def _single_line(value: Any) -> str:
+    return _LINE_BREAKS.sub(" ", _string(value))
+
+
+def _normalize_line_breaks(value: str) -> str:
+    return _LINE_BREAKS.sub("\n", value)
+
+
+def _longest_backtick_run(value: str) -> int:
+    return max((len(match.group(0)) for match in _BACKTICK_RUNS.finditer(value)), default=0)
+
+
+def _underscore_is_inside_identifier(value: str, index: int) -> bool:
+    return (
+        index > 0
+        and index + 1 < len(value)
+        and value[index - 1].isalnum()
+        and value[index + 1].isalnum()
+    )
+
+
+def _string(value: Any) -> str:
+    return "" if value is None else str(value)

agentloop/metrics.py

@@ -0,0 +1,123 @@
+from __future__ import annotations
+
+from collections import Counter, defaultdict
+from typing import Any
+
+from agentloop.costs import estimate_cost_usd
+from agentloop.timing import cumulative_span_time_ms, elapsed_runtime_ms
+
+
+def build_report(trace: Any) -> dict[str, Any]:
+    events = trace.events
+    model_events = [e for e in events if e.event_type == "model_call"]
+    tool_events = [e for e in events if e.event_type == "tool_call"]
+    retry_events = [e for e in events if e.event_type == "retry"]
+
+    repeated = repeated_context_stats(model_events)
+    parallel = parallelism_opportunities(tool_events)
+    cumulative_time = cumulative_span_time_ms(events)
+
+    return {
+        "run_id": trace.run_id,
+        "name": trace.name,
+        "event_count": len(events),
+        "model_call_count": len(model_events),
+        "tool_call_count": len(tool_events),
+        "retry_count": len(retry_events),
+        "total_runtime_ms": round(elapsed_runtime_ms(trace), 3),
+        "cumulative_span_time_ms": round(cumulative_time, 3),
+        "model_time_ms": round(sum(e.duration_ms for e in model_events), 3),
+        "tool_time_ms": round(sum(e.duration_ms for e in tool_events), 3),
+        "retry_time_ms": round(sum(e.duration_ms for e in retry_events), 3),
+        "input_tokens": sum(e.input_tokens for e in model_events),
+        "output_tokens": sum(e.output_tokens for e in model_events),
+        "estimated_cost_usd": round(
+            sum(estimate_cost_usd(e.model, e.input_tokens, e.output_tokens) for e in model_events),
+            6,
+        ),
+        "repeated_context_tokens": repeated["repeated_context_tokens"],
+        "repeated_context_ratio": repeated["repeated_context_ratio"],
+        "parallelism_opportunities": parallel,
+        "recommendations": build_recommendations(model_events, retry_events, repeated, parallel),
+        "events": [e.to_dict() for e in events],
+    }
+
+
+def repeated_context_stats(model_events: list[Any]) -> dict[str, Any]:
+    prefixes = []
+    total_tokens = sum(e.input_tokens for e in model_events)
+    for event in model_events:
+        words = (event.input_text or "").split()
+        if words:
+            prefixes.append((" ".join(words[:80]), min(len(words), 80)))
+    counts = Counter(prefix for prefix, _ in prefixes)
+    repeated_tokens = sum(tokens for prefix, tokens in prefixes if counts[prefix] > 1)
+    avoidable = max(0, repeated_tokens // 2)
+    ratio = avoidable / total_tokens if total_tokens else 0.0
+    return {"repeated_context_tokens": avoidable, "repeated_context_ratio": round(ratio, 4)}
+
+
+def parallelism_opportunities(tool_events: list[Any]) -> list[dict[str, Any]]:
+    grouped: dict[str, list[Any]] = defaultdict(list)
+    for event in tool_events:
+        grouped[event.name].append(event)
+    out = []
+    for name, items in grouped.items():
+        if len(items) >= 3:
+            sequential = sum(item.duration_ms for item in items)
+            parallel = max(item.duration_ms for item in items)
+            out.append(
+                {
+                    "tool_name": name,
+                    "count": len(items),
+                    "sequential_time_ms": round(sequential, 3),
+                    "estimated_parallel_time_ms": round(parallel, 3),
+                    "estimated_savings_ms": round(max(0.0, sequential - parallel), 3),
+                }
+            )
+    return out
+
+
+def build_recommendations(
+    model_events: list[Any],
+    retry_events: list[Any],
+    repeated: dict[str, Any],
+    parallel: list[dict[str, Any]],
+) -> list[dict[str, str]]:
+    recs = []
+    if repeated["repeated_context_ratio"] >= 0.10:
+        recs.append(
+            {
+                "title": "Cache repeated context",
+                "description": "Stable instructions appear across multiple model calls. Use cached prefixes or reusable summaries.",
+            }
+        )
+    if parallel:
+        recs.append(
+            {
+                "title": "Parallelize tool calls",
+                "description": "Repeated tool calls appear independent. Run them concurrently to lower end-to-end latency.",
+            }
+        )
+    if retry_events:
+        recs.append(
+            {
+                "title": "Use structured outputs",
+                "description": "Retries were recorded. Add schema validation or smaller repair prompts.",
+            }
+        )
+    if model_events:
+        largest = max(model_events, key=lambda e: e.input_tokens + e.output_tokens)
+        if largest.total_tokens >= 4000:
+            recs.append(
+                {
+                    "title": "Compress largest model step",
+                    "description": f"The {largest.name} step used {largest.total_tokens} tokens. Consider staged summarization.",
+                }
+            )
+    return recs or [
+        {
+            "title": "No major pattern detected",
+            "description": "Collect more traces for stronger recommendations.",
+        }
+    ]