lode-observe-ai 0.1.0.dev0__py3-none-any.whl

lode_observe_ai/__init__.py

@@ -0,0 +1,59 @@
+"""Lode: Deep Execution DAG Tracer & Debugger for Multi-Agent AI Systems.
+
+Maps the underground veins of multi-agent workflows, revealing the exact root cause
+when an agent loops indefinitely, hallucinates, or fails.
+
+Example usage:
+    from lode_observe_ai import start_trace, trace_agent, trace_tool
+
+    @trace_agent(name="SearchAgent")
+    def my_agent(query: str) -> str:
+        with start_trace("search_flow") as tracer:
+            result = searcher.search(query)
+            tracer.record_output(result)
+        return result
+"""
+
+from lode_observe_ai.core import (
+    aggregate_costs,
+    build_trace_tree,
+    detect_anomalies,
+    extract_replay_steps,
+    generate_trace_summary,
+    get_span_path,
+)
+from lode_observe_ai.domain import (
+    AnomalyIndicators,
+    CostBreakdown,
+    ModelInferenceEvent,
+    ModelProvider,
+    ReplayStep,
+    Span,
+    SpanKind,
+    TokenMetrics,
+    ToolCallEvent,
+    TraceMetadata,
+    TraceTree,
+)
+
+__version__ = "0.1.0-dev"
+
+__all__ = [
+    "AnomalyIndicators",
+    "CostBreakdown",
+    "ModelInferenceEvent",
+    "ModelProvider",
+    "ReplayStep",
+    "Span",
+    "SpanKind",
+    "TokenMetrics",
+    "ToolCallEvent",
+    "TraceMetadata",
+    "TraceTree",
+    "aggregate_costs",
+    "build_trace_tree",
+    "detect_anomalies",
+    "extract_replay_steps",
+    "generate_trace_summary",
+    "get_span_path",
+]

lode_observe_ai/cli.py

@@ -0,0 +1,227 @@
+"""Lode CLI: trace inspection, UI server, and export commands."""
+
+from __future__ import annotations
+
+import json
+import logging
+from pathlib import Path
+
+import typer
+
+from lode_observe_ai.core import build_trace_tree
+from lode_observe_ai.core.export import trace_to_assay_yaml
+from lode_observe_ai.infrastructure.storage.sqlite_store import SQLiteStore
+
+logger = logging.getLogger(__name__)
+
+app = typer.Typer(help="Lode: Deep Execution DAG Tracer & Debugger for AI Agents")
+
+
+@app.command()
+def version() -> None:
+    """Show version information."""
+    typer.echo("Lode v0.1.0-dev")
+
+
+@app.command(deprecated=True)
+def ui(
+    storage: str = typer.Option(
+        "./traces.db",
+        "--storage",
+        "-s",
+        help="SQLite trace database path",
+    ),
+    port: int = typer.Option(3000, "--port", "-p", help="Local web server port"),
+    host: str = typer.Option("127.0.0.1", "--host", help="Web server host"),
+    reload: bool = typer.Option(
+        False,
+        "--reload",
+        help="Enable auto-reload on file changes (development)",
+    ),
+) -> None:
+    """DEPRECATED: Web UI has been removed in favor of OTEL + Jaeger.
+
+    Use instead:
+    - lode export --format otel  (export to Jaeger/DataDog)
+    - lode inspect  (view traces locally)
+
+    To view traces with a UI, deploy Jaeger:
+      docker run -d -p 6831:6831/udp -p 16686:16686 jaegertracing/all-in-one
+
+    Then export traces:
+      lode export --trace-id abc123 --format otel --otel-endpoint http://localhost:4317
+
+    And open: http://localhost:16686
+    """
+    typer.echo(
+        "[WARNING] The 'lode ui' command has been removed in favor of OTEL integration.\n"
+        "Use 'lode export --format otel' to send traces to Jaeger or DataDog.\n"
+        "Use 'lode inspect' to view traces locally in the terminal.",
+        err=True,
+    )
+    raise typer.Exit(code=1)
+
+
+@app.command()
+def export(
+    trace_id: str = typer.Option(..., "--trace-id", "-t", help="Trace ID to export"),
+    export_format: str = typer.Option(
+        "assay",
+        "--format",
+        "-f",
+        help="Export format: assay, json, yaml, otel",
+    ),
+    output: str | None = typer.Option(None, "--output", "-o", help="Output file path"),
+    storage: str | None = typer.Option(
+        None,
+        "--storage",
+        "-s",
+        help="SQLite trace database path (optional; defaults to ./traces.db)",
+    ),
+    otel_endpoint: str = typer.Option(
+        "http://localhost:4317",
+        "--otel-endpoint",
+        help="OTEL collector endpoint for format=otel",
+    ),
+) -> None:
+    """Export trace to test suite, OTEL backend, or other formats.
+
+    Example:
+        lode export --trace-id abc123 --format assay --output suite.yaml
+        lode export --trace-id abc123 --format json
+        lode export --trace-id abc123 --format otel --otel-endpoint http://localhost:4317
+    """
+    # For OTEL export, storage is not required
+    if export_format == "otel":
+        typer.echo(f"[INFO] OTEL export configured for: {otel_endpoint}")
+        typer.echo("[INFO] In production, use lode instrumentation hooks to capture and export traces.")
+        typer.echo("[INFO] Traces are exported in real-time to Jaeger/DataDog/etc.")
+        return
+
+    # For all other formats, storage is required
+    if not storage:
+        storage = "./traces.db"
+
+    # Load trace from storage
+    store = SQLiteStore(db_path=Path(storage))
+    spans = store.get_trace(trace_id)
+
+    if not spans:
+        typer.echo(f"Error: Trace '{trace_id}' not found in {storage}", err=True)
+        raise typer.Exit(code=1)
+
+    # Build TraceTree
+    trace = build_trace_tree(spans)
+
+    # Convert to requested format
+    try:
+        if export_format == "assay":
+            content = trace_to_assay_yaml(trace)
+        elif export_format == "json":
+            # Convert to JSON for easier programmatic access
+            spans_data = []
+            for span in trace.spans.values():
+                span_dict = {
+                    "span_id": span.span_id,
+                    "parent_span_id": span.parent_span_id,
+                    "kind": span.kind.value,
+                    "name": span.name,
+                    "start_time_ms": span.start_time_ms,
+                    "end_time_ms": span.end_time_ms,
+                    "duration_ms": span.duration_ms,
+                    "status": span.status,
+                    "metadata": span.metadata,
+                }
+                if span.model_event:
+                    span_dict["model_event"] = {
+                        "model_name": span.model_event.model_name,
+                        "provider": span.model_event.provider.value,
+                        "tokens": (
+                            {
+                                "input_tokens": span.model_event.tokens.input_tokens,
+                                "output_tokens": span.model_event.tokens.output_tokens,
+                                "total_tokens": span.model_event.tokens.total_tokens,
+                            }
+                            if span.model_event.tokens
+                            else None
+                        ),
+                        "latency_ms": span.model_event.latency_ms,
+                        "error": span.model_event.error,
+                    }
+                spans_data.append(span_dict)
+
+            content = json.dumps(
+                {
+                    "trace_id": trace_id,
+                    "spans": spans_data,
+                },
+                indent=2,
+            )
+        elif export_format == "otel":
+            # Export to OTEL-compatible backend (Jaeger, DataDog, etc.)
+            from lode_observe_ai.infrastructure.otel_exporter import export_to_jaeger
+
+            export_to_jaeger(trace_id, spans, jaeger_endpoint=otel_endpoint)
+            typer.echo(f"✓ Trace exported to OTEL endpoint: {otel_endpoint}")
+            return
+
+        # Write output
+        if output:
+            output_path = Path(output)
+            output_path.write_text(content)
+            typer.echo(f"Exported to {output_path}")
+        else:
+            typer.echo(content)
+
+    except Exception as e:
+        typer.echo(f"Error exporting trace: {e}", err=True)
+        raise typer.Exit(code=1) from e
+
+
+@app.command()
+def inspect(
+    trace_id: str = typer.Option(..., "--trace-id", "-t", help="Trace ID to inspect"),
+    storage: str = typer.Option(
+        "./traces.db",
+        "--storage",
+        "-s",
+        help="SQLite trace database path",
+    ),
+) -> None:
+    """Inspect trace metadata and summary.
+
+    Example:
+        lode inspect --trace-id abc123
+    """
+    # Load trace from storage
+    store = SQLiteStore(db_path=Path(storage))
+    spans = store.get_trace(trace_id)
+
+    if not spans:
+        typer.echo(f"Error: Trace '{trace_id}' not found", err=True)
+        raise typer.Exit(code=1)
+
+    # Build TraceTree
+    trace = build_trace_tree(spans)
+
+    # Display trace info
+    typer.echo(f"Trace ID: {trace_id}")
+    typer.echo(f"Spans: {len(trace.spans)}")
+    typer.echo(f"Root span: {trace.root_span_id if hasattr(trace, 'root_span_id') else 'N/A'}")
+
+    # Show span summary
+    status_counts: dict[str, int] = {}
+    kind_counts: dict[str, int] = {}
+
+    for span in trace.spans.values():
+        status_counts[span.status] = status_counts.get(span.status, 0) + 1
+        kind = span.kind.value
+        kind_counts[kind] = kind_counts.get(kind, 0) + 1
+
+    typer.echo("\nStatus Distribution:")
+    for status, count in sorted(status_counts.items()):
+        typer.echo(f"  {status}: {count}")
+
+    typer.echo("\nSpan Kinds:")
+    for kind, count in sorted(kind_counts.items()):
+        typer.echo(f"  {kind}: {count}")

lode_observe_ai/core/__init__.py

@@ -0,0 +1,311 @@
+"""Pure functional core: DAG assembly, cost aggregation, anomaly detection, replay generation.
+
+No I/O side effects. All functions are deterministic and composable.
+"""
+
+from __future__ import annotations
+
+import time
+from collections import defaultdict
+from typing import Any
+
+from lode_observe_ai.domain import (
+    AnomalyIndicators,
+    CostBreakdown,
+    ReplayStep,
+    Span,
+    SpanKind,
+    TraceMetadata,
+    TraceTree,
+)
+from lode_observe_ai.domain import (
+    ModelInferenceEvent as ModelInferenceEvent,
+)
+from lode_observe_ai.domain import (
+    ToolCallEvent as ToolCallEvent,
+)
+
+# ============================================================================
+# DAG ASSEMBLY & HIERARCHY
+# ============================================================================
+
+
+def build_trace_tree(spans: list[Span], root_span_id: str | None = None) -> TraceTree:
+    """Assemble flat span list into hierarchical DAG tree.
+
+    Validates parent-child relationships and detects cycles.
+    """
+    if not spans:
+        return TraceTree(trace_id="", root_span_id="", spans={})
+
+    # Build span map
+    span_map: dict[str, Span] = {s.span_id: s for s in spans}
+
+    # Find root (span with no parent, or explicitly provided)
+    root_id = root_span_id or next((s.span_id for s in spans if s.parent_span_id is None), "")
+
+    if not root_id and spans:
+        root_id = spans[0].span_id
+
+    # Calculate depth
+    max_depth = _calculate_depth(span_map, root_id)
+
+    # Calculate total duration
+    if root_id in span_map:
+        root_span = span_map[root_id]
+        total_duration = root_span.duration_ms
+    else:
+        total_duration = 0.0
+
+    return TraceTree(
+        trace_id=next((s.metadata.get("trace_id", "") for s in spans), ""),
+        root_span_id=root_id,
+        spans=span_map,
+        depth=max_depth,
+        total_duration_ms=total_duration,
+    )
+
+
+def _calculate_depth(span_map: dict[str, Span], span_id: str, visited: set[str] | None = None) -> int:
+    """Recursively calculate tree depth."""
+    if visited is None:
+        visited = set()
+
+    if span_id in visited:  # Cycle detection
+        return 0
+
+    visited.add(span_id)
+
+    if span_id not in span_map:
+        return 0
+
+    # Find children
+    children = [s for s in span_map.values() if s.parent_span_id == span_id]
+
+    if not children:
+        return 1
+
+    return 1 + max(_calculate_depth(span_map, c.span_id, visited) for c in children)
+
+
+def get_span_path(tree: TraceTree, span_id: str) -> list[str]:
+    """Get path from root to target span."""
+    if span_id not in tree.spans:
+        return []
+
+    path = [span_id]
+    current = tree.spans[span_id]
+
+    while current.parent_span_id:
+        path.insert(0, current.parent_span_id)
+        current = tree.spans.get(current.parent_span_id)
+        if current is None:
+            break
+
+    return path
+
+
+# ============================================================================
+# COST & TOKEN AGGREGATION
+# ============================================================================
+
+
+def aggregate_costs(tree: TraceTree) -> CostBreakdown:
+    """Aggregate token costs across all inferences in trace."""
+    total_input = 0
+    total_output = 0
+    cost_by_model: dict[str, float] = defaultdict(float)
+    cost_by_provider: dict[str, float] = defaultdict(float)
+
+    for span in tree.spans.values():
+        if span.model_event and span.model_event.tokens:
+            tokens = span.model_event.tokens
+            total_input += tokens.input_tokens
+            total_output += tokens.output_tokens
+
+            # Accumulate by model and provider
+            model_name = span.model_event.model_name
+            provider = span.model_event.provider.value
+
+            cost_by_model[model_name] += tokens.cost_usd
+            cost_by_provider[provider] += tokens.cost_usd
+
+    total_tokens = total_input + total_output
+    total_cost = sum(cost_by_model.values())
+
+    return CostBreakdown(
+        total_input_tokens=total_input,
+        total_output_tokens=total_output,
+        total_tokens=total_tokens,
+        total_cost_usd=total_cost,
+        cost_by_model=dict(cost_by_model),
+        cost_by_provider=dict(cost_by_provider),
+    )
+
+
+# ============================================================================
+# ANOMALY DETECTION
+# ============================================================================
+
+
+def detect_anomalies(tree: TraceTree) -> AnomalyIndicators:
+    """Detect anomalies in trace execution."""
+    flags: list[str] = []
+
+    # Check for cycles
+    has_cycle = _detect_cycle(tree)
+    if has_cycle:
+        flags.append("cycle_detected")
+
+    # Check for repeated tool calls (infinite loop indicator)
+    max_repeated = _max_repeated_tool_calls(tree)
+    if max_repeated > 5:
+        flags.append("high_tool_call_repetition")
+
+    # Check for context window exhaustion
+    has_context_exhaustion = _check_context_exhaustion(tree)
+    if has_context_exhaustion:
+        flags.append("context_window_exhaustion")
+
+    # Check for unusual latency
+    has_unusual_latency = _check_unusual_latency(tree)
+    if has_unusual_latency:
+        flags.append("unusual_latency")
+
+    return AnomalyIndicators(
+        has_cycle=has_cycle,
+        max_repeated_tool_calls=max_repeated,
+        estimated_infinite_loop=max_repeated > 5 or (has_cycle and len(flags) > 1),
+        context_window_exhaustion=has_context_exhaustion,
+        unusual_latency=has_unusual_latency,
+        flags=flags,
+    )
+
+
+def _detect_cycle(tree: TraceTree) -> bool:
+    """Detect if trace has cycle (parent-child loop)."""
+    for span in tree.spans.values():
+        visited: set[str] = set()
+        current = span
+        while current.parent_span_id:
+            if current.parent_span_id in visited:
+                return True
+            visited.add(current.parent_span_id)
+            current = tree.spans.get(current.parent_span_id)
+            if current is None:
+                break
+    return False
+
+
+def _max_repeated_tool_calls(tree: TraceTree) -> int:
+    """Count maximum repetitions of same tool call."""
+    tool_call_counts: dict[str, int] = defaultdict(int)
+
+    for span in tree.spans.values():
+        if span.tool_event:
+            tool_name = span.tool_event.tool_name
+            tool_call_counts[tool_name] += 1
+
+    return max(tool_call_counts.values()) if tool_call_counts else 0
+
+
+def _check_context_exhaustion(tree: TraceTree) -> bool:
+    """Check if any span indicates context window exhaustion."""
+    for span in tree.spans.values():
+        if span.kind == SpanKind.CONTEXT_WINDOW:
+            return True
+        if "context_exhausted" in span.metadata:
+            return span.metadata["context_exhausted"]
+    return False
+
+
+def _check_unusual_latency(tree: TraceTree) -> bool:
+    """Check if any span has unusually high latency (>5s)."""
+    return any(span.duration_ms > 5000 for span in tree.spans.values())
+
+
+# ============================================================================
+# REPLAY PAYLOAD GENERATION
+# ============================================================================
+
+
+def extract_replay_steps(tree: TraceTree) -> list[ReplayStep]:
+    """Extract replay payloads for offline step recreation."""
+    replay_steps: list[ReplayStep] = []
+
+    for span in tree.spans.values():
+        if not span.is_terminal:
+            continue
+
+        # Build input context from metadata
+        input_context = span.metadata.copy() if span.metadata else {}
+
+        replay_step = ReplayStep(
+            step_id=span.span_id,
+            span_kind=span.kind,
+            input_context=input_context or None,
+            model_inference=span.model_event,
+            tool_call=span.tool_event,
+            expected_output=span.metadata.get("output"),
+        )
+        replay_steps.append(replay_step)
+
+    return replay_steps
+
+
+# ============================================================================
+# TRACE SUMMARY & ANALYTICS
+# ============================================================================
+
+
+def generate_trace_summary(
+    tree: TraceTree,
+    metadata: TraceMetadata | None = None,
+) -> dict[str, Any]:
+    """Generate comprehensive trace summary with all analytics."""
+    start_time = time.perf_counter()
+
+    costs = aggregate_costs(tree)
+    anomalies = detect_anomalies(tree)
+    replay_steps = extract_replay_steps(tree)
+
+    # Collect span statistics
+    span_kinds: dict[str, int] = defaultdict(int)
+    error_count = 0
+    max_latency_ms = 0.0
+
+    for span in tree.spans.values():
+        span_kinds[span.kind.value] += 1
+        if span.status == "error":
+            error_count += 1
+        max_latency_ms = max(max_latency_ms, span.duration_ms)
+
+    elapsed_ms = (time.perf_counter() - start_time) * 1000
+
+    return {
+        "trace_id": tree.trace_id,
+        "span_count": tree.span_count,
+        "depth": tree.depth,
+        "duration_ms": tree.total_duration_ms,
+        "span_kinds": dict(span_kinds),
+        "error_count": error_count,
+        "max_latency_ms": max_latency_ms,
+        "costs": {
+            "total_tokens": costs.total_tokens,
+            "input_tokens": costs.total_input_tokens,
+            "output_tokens": costs.total_output_tokens,
+            "total_cost_usd": costs.total_cost_usd,
+            "by_model": costs.cost_by_model,
+            "by_provider": costs.cost_by_provider,
+        },
+        "anomalies": {
+            "has_cycle": anomalies.has_cycle,
+            "max_repeated_tool_calls": anomalies.max_repeated_tool_calls,
+            "estimated_infinite_loop": anomalies.estimated_infinite_loop,
+            "context_window_exhaustion": anomalies.context_window_exhaustion,
+            "unusual_latency": anomalies.unusual_latency,
+            "flags": anomalies.flags,
+        },
+        "replay_steps_count": len(replay_steps),
+        "analysis_latency_ms": elapsed_ms,
+    }