fableforge-agent-telemetry 0.1.0__py3-none-any.whl

agent_telemetry/__init__.py

@@ -0,0 +1,7 @@
+"""AgentTelemetry — Datadog for AI agents."""
+
+__version__ = "0.1.0"
+
+from agent_telemetry.models import Span, SessionMetrics, ToolMetrics, CostReport
+
+__all__ = ["Span", "SessionMetrics", "ToolMetrics", "CostReport", "__version__"]

agent_telemetry/cli.py

@@ -0,0 +1,212 @@
+"""CLI for AgentTelemetry — analyze traces, view costs, start dashboard."""
+
+from __future__ import annotations
+
+import sys
+from pathlib import Path
+from typing import Optional
+
+import click
+from rich.console import Console
+from rich.table import Table
+
+from agent_telemetry.collector import (
+    auto_detect_format,
+    calculate_metrics,
+    ingest_trace,
+)
+from agent_telemetry.error_tracker import classify_error, generate_error_report
+from agent_telemetry.models import Span
+from agent_telemetry.storage import TelemetryStorage
+from agent_telemetry.token_tracker import estimate_cost, format_cost_table
+
+console = Console()
+
+
+@click.group()
+@click.version_option(version="0.1.0")
+def cli() -> None:
+    """AgentTelemetry — Datadog for AI agents."""
+
+
+@cli.command()
+@click.argument("trace_file", type=click.Path(exists=True))
+@click.option("--format", "fmt", type=click.Choice(["glint", "armand0e", "vfable", "auto"]), default="auto", help="Trace format")
+@click.option("--store/--no-store", default=True, help="Store results in database")
+def analyze(trace_file: str, fmt: str, store: bool) -> None:
+    """Analyze a trace file and display metrics."""
+    if fmt == "auto":
+        fmt = auto_detect_format(trace_file)
+        console.print(f"[dim]Detected format: {fmt}[/dim]")
+
+    spans = ingest_trace(trace_file, fmt=fmt)
+    if not spans:
+        console.print("[red]No spans found in trace file.[/red]")
+        sys.exit(1)
+
+    console.print(f"[green]Loaded {len(spans)} spans[/green]")
+
+    metrics_result = calculate_metrics(spans)
+    session = metrics_result["session"]
+    tools = metrics_result["tools"]
+
+    console.print(f"\n[bold]Session:[/bold] {session.session_id}")
+    console.print(f"[bold]Model:[/bold]    {session.model}")
+    console.print(f"[bold]Duration:[/bold]  {session.duration_seconds:.1f}s")
+
+    metrics_table = Table(title="Session Metrics", show_header=True)
+    metrics_table.add_column("Metric", style="cyan")
+    metrics_table.add_column("Value", justify="right")
+
+    metrics_table.add_row("Total Tokens", f"{session.total_tokens:,}")
+    metrics_table.add_row("Total Cost", f"${session.total_cost:.6f}")
+    metrics_table.add_row("Tool Calls", str(session.tool_calls))
+    metrics_table.add_row("Errors", str(session.error_count))
+    metrics_table.add_row("Avg Duration", f"{session.avg_tool_duration_ms:.0f}ms")
+    metrics_table.add_row("P50 Duration", f"{session.p50_duration_ms:.0f}ms")
+    metrics_table.add_row("P95 Duration", f"{session.p95_duration_ms:.0f}ms")
+    metrics_table.add_row("P99 Duration", f"{session.p99_duration_ms:.0f}ms")
+    metrics_table.add_row("Cache Hit Rate", f"{session.cache_hit_rate:.1%}")
+    console.print(metrics_table)
+
+    tool_table = Table(title="Tool Metrics", show_header=True)
+    tool_table.add_column("Tool", style="cyan")
+    tool_table.add_column("Calls", justify="right")
+    tool_table.add_column("Avg ms", justify="right")
+    tool_table.add_column("P95 ms", justify="right")
+    tool_table.add_column("Error Rate", justify="right")
+    tool_table.add_column("Cost", justify="right", style="green")
+
+    for name, tm in sorted(tools.items()):
+        tool_table.add_row(
+            name,
+            str(tm.call_count),
+            f"{tm.avg_duration_ms:.0f}",
+            f"{tm.p95_duration_ms:.0f}",
+            f"{tm.error_rate:.1%}",
+            f"${tm.total_cost_usd:.6f}",
+        )
+    console.print(tool_table)
+
+    if store:
+        storage = TelemetryStorage()
+        storage.store_spans(spans)
+        storage.store_session_metrics(session)
+        console.print(f"\n[dim]Stored {len(spans)} spans in database[/dim]")
+
+
+@cli.command()
+@click.option("--host", default="127.0.0.1", help="Host to bind to")
+@click.option("--port", default=8088, type=int, help="Port to bind to")
+def dashboard(host: str, port: int) -> None:
+    """Start the interactive dashboard server."""
+    import uvicorn
+    from agent_telemetry.dashboard import app
+
+    console.print(f"[green]Starting AgentTelemetry dashboard on http://{host}:{port}[/green]")
+    console.print("[dim]Press Ctrl+C to stop[/dim]")
+    uvicorn.run(app, host=host, port=port)
+
+
+@cli.command()
+@click.argument("trace_file", type=click.Path(exists=True))
+@click.option("--format", "fmt", type=click.Choice(["glint", "armand0e", "vfable", "auto"]), default="auto", help="Trace format")
+def cost(trace_file: str, fmt: str) -> None:
+    """Show cost breakdown for a trace file."""
+    if fmt == "auto":
+        fmt = auto_detect_format(trace_file)
+        console.print(f"[dim]Detected format: {fmt}[/dim]")
+
+    spans = ingest_trace(trace_file, fmt=fmt)
+    if not spans:
+        console.print("[red]No spans found in trace file.[/red]")
+        sys.exit(1)
+
+    models: dict[str, list[Span]] = {}
+    for s in spans:
+        models.setdefault(s.model, []).append(s)
+
+    breakdowns = []
+    for model, model_spans in sorted(models.items()):
+        bd = estimate_cost(
+            sum(s.input_tokens for s in model_spans),
+            sum(s.output_tokens for s in model_spans),
+            model,
+            sum(s.cache_read for s in model_spans),
+            sum(s.cache_creation for s in model_spans),
+        )
+        breakdowns.append(bd)
+
+    console.print(format_cost_table(breakdowns))
+
+    total = sum(b.total_cost for b in breakdowns)
+    console.print(f"\n[bold green]Grand Total: ${total:.6f}[/bold green]")
+
+
+@cli.command()
+@click.argument("trace_file", type=click.Path(exists=True))
+@click.option("--format", "fmt", type=click.Choice(["glint", "armand0e", "vfable", "auto"]), default="auto", help="Trace format")
+def errors(trace_file: str, fmt: str) -> None:
+    """Show error report for a trace file."""
+    if fmt == "auto":
+        fmt = auto_detect_format(trace_file)
+        console.print(f"[dim]Detected format: {fmt}[/dim]")
+
+    spans = ingest_trace(trace_file, fmt=fmt)
+    if not spans:
+        console.print("[red]No spans found in trace file.[/red]")
+        sys.exit(1)
+
+    session_id = spans[0].session_id
+    report = generate_error_report(session_id, spans=spans)
+
+    console.print(f"\n[bold]Error Report: {session_id}[/bold]")
+    console.print(f"Total Errors:  {report.total_errors}")
+    console.print(f"Recovered:      {report.recovered_errors}")
+    console.print(f"Recovery Rate:  {report.recovery_rate:.0%}")
+
+    if report.errors_by_type:
+        type_table = Table(title="Errors by Type", show_header=True)
+        type_table.add_column("Error Type", style="red")
+        type_table.add_column("Count", justify="right")
+
+        for etype, count in sorted(report.errors_by_type.items(), key=lambda x: -x[1]):
+            type_table.add_row(etype, str(count))
+        console.print(type_table)
+
+    if report.errors:
+        error_table = Table(title="Error Details", show_header=True)
+        error_table.add_column("Span ID", style="dim")
+        error_table.add_column("Type", style="red")
+        error_table.add_column("Tool", style="cyan")
+        error_table.add_column("Message", max_width=60)
+        error_table.add_column("Recovered")
+
+        for e in report.errors[:50]:
+            recovered = "[green]✓[/green]" if e.recovered else "[red]✗[/red]"
+            error_table.add_row(
+                e.span_id[:12] + "...",
+                e.error_type,
+                e.tool_name,
+                e.error_message[:60],
+                recovered,
+            )
+        console.print(error_table)
+
+
+@cli.command()
+@click.argument("text")
+@click.option("--model", default="gpt-4", help="Model name for token counting")
+def tokens(text: str, model: str) -> None:
+    """Count tokens in a text string."""
+    from agent_telemetry.token_tracker import count_tokens
+
+    n = count_tokens(text, model)
+    console.print(f"[bold]{n:,}[/bold] tokens ({model})")
+
+    bd = estimate_cost(n, 0, model)
+    console.print(f"Input cost (no output): ${bd.input_cost:.6f}")
+
+
+if __name__ == "__main__":
+    cli()

agent_telemetry/collector.py

@@ -0,0 +1,412 @@
+"""Trace data ingestion: parse multiple agent trace formats and extract spans."""
+
+from __future__ import annotations
+
+import json
+import statistics
+from datetime import datetime
+from pathlib import Path
+from typing import Any
+
+from agent_telemetry.models import (
+    CostReport,
+    SessionMetrics,
+    Span,
+    SpanStatus,
+    ToolMetrics,
+)
+from agent_telemetry.token_tracker import estimate_cost
+
+
+def parse_glint_trace(jsonl_path: str | Path) -> list[dict[str, Any]]:
+    """Parse Glint-Research format trace files.
+
+    Expected format per line:
+    {
+        "type": "tool_call" | "tool_result" | "message" | "error",
+        "timestamp": "2025-01-15T10:30:00Z",
+        "session_id": "...",
+        "span_id": "...",
+        "tool": "Bash" | "Read" | "Edit" | "Write" | ...,
+        "input": { ... },
+        "output": { ... },
+        "usage": {
+            "input_tokens": 1234,
+            "output_tokens": 567,
+            "cache_read_input_tokens": 100,
+            "cache_creation_input_tokens": 50
+        },
+        "duration_ms": 1234.5,
+        "model": "claude-3.5-sonnet",
+        "error": null | "error message"
+    }
+    """
+    spans: list[dict[str, Any]] = []
+    path = Path(jsonl_path)
+
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            if entry.get("type") not in ("tool_call", "message"):
+                continue
+
+            usage = entry.get("usage", {})
+            span = {
+                "span_id": entry.get("span_id", ""),
+                "session_id": entry.get("session_id", "unknown"),
+                "tool_name": entry.get("tool", entry.get("type", "unknown")),
+                "input_tokens": usage.get("input_tokens", 0),
+                "output_tokens": usage.get("output_tokens", 0),
+                "cache_read": usage.get("cache_read_input_tokens", 0),
+                "cache_creation": usage.get("cache_creation_input_tokens", 0),
+                "duration_ms": entry.get("duration_ms", 0.0),
+                "status": SpanStatus.ERROR if entry.get("error") else SpanStatus.OK,
+                "error": entry.get("error"),
+                "model": entry.get("model", "unknown"),
+                "timestamp": entry.get("timestamp"),
+                "metadata": {},
+            }
+            cost = estimate_cost(
+                span["input_tokens"],
+                span["output_tokens"],
+                span["model"],
+                span["cache_read"],
+                span["cache_creation"],
+            )
+            span["cost_usd"] = cost.total_cost
+            spans.append(span)
+
+    return spans
+
+
+def parse_armand0e_trace(jsonl_path: str | Path) -> list[dict[str, Any]]:
+    """Parse armand0e format trace files.
+
+    Expected format per line:
+    {
+        "event": "invocation" | "response" | "error",
+        "id": "span-id",
+        "session": "session-id",
+        "timestamp": "2025-01-15T10:30:00Z",
+        "tool": {"name": "Bash", "input": {...}},
+        "result": {...},
+        "tokens": {"in": 1234, "out": 567, "cached": 100},
+        "latency_ms": 1234.5,
+        "model": "gpt-4o",
+        "error": null | "..."
+    }
+    """
+    spans: list[dict[str, Any]] = []
+    path = Path(jsonl_path)
+    pending: dict[str, dict[str, Any]] = {}
+
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            event = entry.get("event", "")
+            span_id = entry.get("id", "")
+
+            if event == "invocation":
+                pending[span_id] = {
+                    "span_id": span_id,
+                    "session_id": entry.get("session", "unknown"),
+                    "tool_name": entry.get("tool", {}).get("name", "unknown"),
+                    "timestamp": entry.get("timestamp"),
+                    "model": entry.get("model", "unknown"),
+                    "metadata": {"input": entry.get("tool", {}).get("input", {})},
+                }
+            elif event in ("response", "error"):
+                tokens = entry.get("tokens", {})
+                span_data = pending.pop(span_id, {
+                    "span_id": span_id,
+                    "session_id": entry.get("session", "unknown"),
+                    "tool_name": "unknown",
+                    "model": entry.get("model", "unknown"),
+                    "timestamp": entry.get("timestamp"),
+                    "metadata": {},
+                })
+
+                span_data["input_tokens"] = tokens.get("in", 0)
+                span_data["output_tokens"] = tokens.get("out", 0)
+                span_data["cache_read"] = tokens.get("cached", 0)
+                span_data["cache_creation"] = tokens.get("cache_write", 0)
+                span_data["duration_ms"] = entry.get("latency_ms", 0.0)
+                span_data["status"] = SpanStatus.ERROR if event == "error" else SpanStatus.OK
+                span_data["error"] = entry.get("error")
+
+                cost = estimate_cost(
+                    span_data["input_tokens"],
+                    span_data["output_tokens"],
+                    span_data["model"],
+                    span_data["cache_read"],
+                    span_data["cache_creation"],
+                )
+                span_data["cost_usd"] = cost.total_cost
+                spans.append(span_data)
+
+    for span_data in pending.values():
+        span_data.setdefault("input_tokens", 0)
+        span_data.setdefault("output_tokens", 0)
+        span_data.setdefault("cache_read", 0)
+        span_data.setdefault("cache_creation", 0)
+        span_data.setdefault("duration_ms", 0.0)
+        span_data.setdefault("status", SpanStatus.OK)
+        span_data.setdefault("error", None)
+        span_data.setdefault("cost_usd", 0.0)
+        spans.append(span_data)
+
+    return spans
+
+
+def parse_vfable_trace(jsonl_path: str | Path) -> list[dict[str, Any]]:
+    """Parse v-Fable format trace files.
+
+    Expected format per line:
+    {
+        "kind": "tool_use" | "tool_result" | "message_start" | "message_end",
+        "timestamp": "2025-01-15T10:30:00Z",
+        "session_id": "...",
+        "span_id": "...",
+        "parent_span_id": null | "...",
+        "tool_name": "Bash",
+        "tokens": {"prompt": 1234, "completion": 567, "cache_read": 100, "cache_write": 50},
+        "duration_ms": 1234.5,
+        "cost_usd": 0.0234,
+        "model": "claude-3.5-sonnet",
+        "status": "success" | "error" | "timeout",
+        "error_message": null | "..."
+    }
+    """
+    spans: list[dict[str, Any]] = []
+    path = Path(jsonl_path)
+
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            if entry.get("kind") not in ("tool_use", "tool_result", "message_end"):
+                continue
+
+            tokens = entry.get("tokens", {})
+            status_str = entry.get("status", "success")
+
+            if status_str == "error":
+                status = SpanStatus.ERROR
+            elif status_str == "timeout":
+                status = SpanStatus.TIMEOUT
+            else:
+                status = SpanStatus.OK
+
+            span = {
+                "span_id": entry.get("span_id", ""),
+                "session_id": entry.get("session_id", "unknown"),
+                "tool_name": entry.get("tool_name", "unknown"),
+                "input_tokens": tokens.get("prompt", 0),
+                "output_tokens": tokens.get("completion", 0),
+                "cache_read": tokens.get("cache_read", 0),
+                "cache_creation": tokens.get("cache_write", 0),
+                "duration_ms": entry.get("duration_ms", 0.0),
+                "cost_usd": entry.get("cost_usd", 0.0),
+                "status": status,
+                "error": entry.get("error_message"),
+                "model": entry.get("model", "unknown"),
+                "timestamp": entry.get("timestamp"),
+                "metadata": {"parent_span_id": entry.get("parent_span_id")},
+            }
+            spans.append(span)
+
+    return spans
+
+
+def extract_spans(trace_data: list[dict[str, Any]]) -> list[Span]:
+    """Convert raw trace dicts into validated Span objects."""
+    spans = []
+    for d in trace_data:
+        ts = d.get("timestamp")
+        if isinstance(ts, str):
+            try:
+                ts = datetime.fromisoformat(ts.replace("Z", "+00:00"))
+            except (ValueError, AttributeError):
+                ts = None
+        elif not isinstance(ts, datetime):
+            ts = None
+
+        status_val = d.get("status", "ok")
+        if isinstance(status_val, str):
+            try:
+                status_val = SpanStatus(status_val)
+            except ValueError:
+                status_val = SpanStatus.OK
+
+        span = Span(
+            span_id=d.get("span_id", ""),
+            session_id=d.get("session_id", "unknown"),
+            tool_name=d.get("tool_name", "unknown"),
+            input_tokens=d.get("input_tokens", 0),
+            output_tokens=d.get("output_tokens", 0),
+            cache_read=d.get("cache_read", 0),
+            cache_creation=d.get("cache_creation", 0),
+            duration_ms=d.get("duration_ms", 0.0),
+            cost_usd=d.get("cost_usd", 0.0),
+            status=status_val,
+            error=d.get("error"),
+            model=d.get("model", "unknown"),
+            timestamp=ts,
+            metadata=d.get("metadata", {}),
+        )
+        spans.append(span)
+    return spans
+
+
+def _percentile(data: list[float], p: float) -> float:
+    if not data:
+        return 0.0
+    sorted_data = sorted(data)
+    k = (len(sorted_data) - 1) * p
+    f = int(k)
+    c = f + 1
+    if c >= len(sorted_data):
+        return sorted_data[-1]
+    return sorted_data[f] + (k - f) * (sorted_data[c] - sorted_data[f])
+
+
+def calculate_metrics(spans: list[Span]) -> dict[str, SessionMetrics | dict[str, ToolMetrics]]:
+    """Calculate session-level and per-tool metrics from spans."""
+    if not spans:
+        return {
+            "session": SessionMetrics(session_id="unknown"),
+            "tools": {},
+        }
+
+    session_id = spans[0].session_id
+    models = {s.model for s in spans}
+    primary_model = next(iter(models)) if len(models) == 1 else "mixed"
+
+    total_input = sum(s.input_tokens for s in spans)
+    total_output = sum(s.output_tokens for s in spans)
+    total_cache_read = sum(s.cache_read for s in spans)
+    total_cache_creation = sum(s.cache_creation for s in spans)
+    total_tokens = total_input + total_output
+    total_cost = sum(s.cost_usd for s in spans)
+    error_count = sum(1 for s in spans if s.status == SpanStatus.ERROR)
+    durations = [s.duration_ms for s in spans if s.duration_ms > 0]
+
+    cache_hit_rate = total_cache_read / total_input if total_input > 0 else 0.0
+
+    timestamps = [s.timestamp for s in spans if s.timestamp]
+    started_at = min(timestamps) if timestamps else None
+    ended_at = max(timestamps) if timestamps else None
+    duration_seconds = 0.0
+    if started_at and ended_at:
+        duration_seconds = (ended_at - started_at).total_seconds()
+
+    session_metrics = SessionMetrics(
+        session_id=session_id,
+        total_tokens=total_tokens,
+        total_cost=total_cost,
+        tool_calls=len(spans),
+        error_count=error_count,
+        recovery_count=0,
+        duration_seconds=duration_seconds,
+        avg_tool_duration_ms=statistics.mean(durations) if durations else 0.0,
+        p50_duration_ms=_percentile(durations, 0.50),
+        p95_duration_ms=_percentile(durations, 0.95),
+        p99_duration_ms=_percentile(durations, 0.99),
+        cache_hit_rate=cache_hit_rate,
+        model=primary_model,
+        started_at=started_at,
+        ended_at=ended_at,
+    )
+
+    tools_by_name: dict[str, list[Span]] = {}
+    for s in spans:
+        tools_by_name.setdefault(s.tool_name, []).append(s)
+
+    tool_metrics: dict[str, ToolMetrics] = {}
+    for tool_name, tool_spans in tools_by_name.items():
+        t_durations = [s.duration_ms for s in tool_spans if s.duration_ms > 0]
+        t_errors = sum(1 for s in tool_spans if s.status == SpanStatus.ERROR)
+
+        tool_metrics[tool_name] = ToolMetrics(
+            tool_name=tool_name,
+            call_count=len(tool_spans),
+            avg_duration_ms=statistics.mean(t_durations) if t_durations else 0.0,
+            p50_duration_ms=_percentile(t_durations, 0.50),
+            p95_duration_ms=_percentile(t_durations, 0.95),
+            error_rate=t_errors / len(tool_spans) if tool_spans else 0.0,
+            total_input_tokens=sum(s.input_tokens for s in tool_spans),
+            total_output_tokens=sum(s.output_tokens for s in tool_spans),
+            total_cache_read=sum(s.cache_read for s in tool_spans),
+            total_cache_creation=sum(s.cache_creation for s in tool_spans),
+            total_cost_usd=sum(s.cost_usd for s in tool_spans),
+            token_distribution={
+                "input": sum(s.input_tokens for s in tool_spans),
+                "output": sum(s.output_tokens for s in tool_spans),
+                "cache_read": sum(s.cache_read for s in tool_spans),
+                "cache_creation": sum(s.cache_creation for s in tool_spans),
+            },
+        )
+
+    return {"session": session_metrics, "tools": tool_metrics}
+
+
+def auto_detect_format(jsonl_path: str | Path) -> str:
+    """Auto-detect the trace format from the first few lines."""
+    path = Path(jsonl_path)
+    with path.open() as f:
+        for line in f:
+            line = line.strip()
+            if not line:
+                continue
+            try:
+                entry = json.loads(line)
+            except json.JSONDecodeError:
+                continue
+
+            if "type" in entry and entry.get("type") in ("tool_call", "message", "tool_result"):
+                return "glint"
+            if "event" in entry and entry.get("event") in ("invocation", "response", "error"):
+                return "armand0e"
+            if "kind" in entry and entry.get("kind") in ("tool_use", "tool_result", "message_start", "message_end"):
+                return "vfable"
+            break
+    return "unknown"
+
+
+def ingest_trace(jsonl_path: str | Path, fmt: str | None = None) -> list[Span]:
+    """Ingest a trace file, auto-detecting format if not specified."""
+    if fmt is None:
+        fmt = auto_detect_format(jsonl_path)
+
+    parsers = {
+        "glint": parse_glint_trace,
+        "armand0e": parse_armand0e_trace,
+        "vfable": parse_vfable_trace,
+    }
+
+    parser = parsers.get(fmt)
+    if parser is None:
+        raise ValueError(f"Unknown trace format: {fmt}. Supported: {list(parsers.keys())}")
+
+    raw_spans = parser(jsonl_path)
+    return extract_spans(raw_spans)