RouteKitAI 0.1.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- routekitai/__init__.py +53 -0
- routekitai/cli/__init__.py +18 -0
- routekitai/cli/main.py +40 -0
- routekitai/cli/replay.py +80 -0
- routekitai/cli/run.py +95 -0
- routekitai/cli/serve.py +966 -0
- routekitai/cli/test_agent.py +178 -0
- routekitai/cli/trace.py +209 -0
- routekitai/cli/trace_analyze.py +120 -0
- routekitai/cli/trace_search.py +126 -0
- routekitai/core/__init__.py +58 -0
- routekitai/core/agent.py +325 -0
- routekitai/core/errors.py +49 -0
- routekitai/core/hooks.py +174 -0
- routekitai/core/memory.py +54 -0
- routekitai/core/message.py +132 -0
- routekitai/core/model.py +91 -0
- routekitai/core/policies.py +373 -0
- routekitai/core/policy.py +85 -0
- routekitai/core/policy_adapter.py +133 -0
- routekitai/core/runtime.py +1403 -0
- routekitai/core/tool.py +148 -0
- routekitai/core/tools.py +180 -0
- routekitai/evals/__init__.py +13 -0
- routekitai/evals/dataset.py +75 -0
- routekitai/evals/metrics.py +101 -0
- routekitai/evals/runner.py +184 -0
- routekitai/graphs/__init__.py +12 -0
- routekitai/graphs/executors.py +457 -0
- routekitai/graphs/graph.py +164 -0
- routekitai/memory/__init__.py +13 -0
- routekitai/memory/episodic.py +242 -0
- routekitai/memory/kv.py +34 -0
- routekitai/memory/retrieval.py +192 -0
- routekitai/memory/vector.py +700 -0
- routekitai/memory/working.py +66 -0
- routekitai/message.py +29 -0
- routekitai/model.py +48 -0
- routekitai/observability/__init__.py +21 -0
- routekitai/observability/analyzer.py +314 -0
- routekitai/observability/exporters/__init__.py +10 -0
- routekitai/observability/exporters/base.py +30 -0
- routekitai/observability/exporters/jsonl.py +81 -0
- routekitai/observability/exporters/otel.py +119 -0
- routekitai/observability/spans.py +111 -0
- routekitai/observability/streaming.py +117 -0
- routekitai/observability/trace.py +144 -0
- routekitai/providers/__init__.py +9 -0
- routekitai/providers/anthropic.py +227 -0
- routekitai/providers/azure_openai.py +243 -0
- routekitai/providers/local.py +196 -0
- routekitai/providers/openai.py +321 -0
- routekitai/py.typed +0 -0
- routekitai/sandbox/__init__.py +12 -0
- routekitai/sandbox/filesystem.py +131 -0
- routekitai/sandbox/network.py +142 -0
- routekitai/sandbox/permissions.py +70 -0
- routekitai/tool.py +33 -0
- routekitai-0.1.0.dist-info/METADATA +328 -0
- routekitai-0.1.0.dist-info/RECORD +64 -0
- routekitai-0.1.0.dist-info/WHEEL +5 -0
- routekitai-0.1.0.dist-info/entry_points.txt +2 -0
- routekitai-0.1.0.dist-info/licenses/LICENSE +21 -0
- routekitai-0.1.0.dist-info/top_level.txt +1 -0
|
@@ -0,0 +1,178 @@
|
|
|
1
|
+
"""CLI command for testing agents."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import typer
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
except ImportError as e:
|
|
11
|
+
raise ImportError("CLI dependencies not installed. Install with: pip install typer rich") from e
|
|
12
|
+
|
|
13
|
+
from routekitai.core.agent import Agent
|
|
14
|
+
from routekitai.core.runtime import Runtime
|
|
15
|
+
from routekitai.core.tools import EchoTool
|
|
16
|
+
from routekitai.providers.local import FakeModel
|
|
17
|
+
|
|
18
|
+
app = typer.Typer(name="test-agent", help="Run sanity checks on routkitai agents")
|
|
19
|
+
console = Console()
|
|
20
|
+
|
|
21
|
+
|
|
22
|
+
def test_command(
|
|
23
|
+
verbose: bool = typer.Option(False, "--verbose", "-v", help="Verbose output"),
|
|
24
|
+
) -> None:
|
|
25
|
+
"""Run a battery of sanity checks on routkitai agents.
|
|
26
|
+
|
|
27
|
+
Tests basic agent functionality, tool execution, tracing, and replay.
|
|
28
|
+
|
|
29
|
+
Examples:
|
|
30
|
+
routkitai test-agent
|
|
31
|
+
routkitai test-agent --verbose
|
|
32
|
+
"""
|
|
33
|
+
|
|
34
|
+
async def _run_tests() -> None:
|
|
35
|
+
console.print("[bold]Running routkitai agent sanity checks...[/bold]\n")
|
|
36
|
+
|
|
37
|
+
tests_passed = 0
|
|
38
|
+
tests_failed = 0
|
|
39
|
+
test_results = []
|
|
40
|
+
|
|
41
|
+
# Test 1: Basic agent creation
|
|
42
|
+
console.print("[cyan]Test 1: Agent creation[/cyan]")
|
|
43
|
+
try:
|
|
44
|
+
model = FakeModel(name="test")
|
|
45
|
+
model.add_response("Hello, I'm a test agent!")
|
|
46
|
+
agent = Agent(name="test_agent", model=model, tools=[EchoTool()])
|
|
47
|
+
test_results.append(("Agent creation", True, ""))
|
|
48
|
+
tests_passed += 1
|
|
49
|
+
if verbose:
|
|
50
|
+
console.print(" [green]✓[/green] Agent created successfully")
|
|
51
|
+
except Exception as e:
|
|
52
|
+
test_results.append(("Agent creation", False, str(e)))
|
|
53
|
+
tests_failed += 1
|
|
54
|
+
console.print(f" [red]✗[/red] Failed: {e}")
|
|
55
|
+
|
|
56
|
+
# Test 2: Agent execution
|
|
57
|
+
console.print("\n[cyan]Test 2: Agent execution[/cyan]")
|
|
58
|
+
try:
|
|
59
|
+
model = FakeModel(name="test")
|
|
60
|
+
model.add_response("Test response")
|
|
61
|
+
agent = Agent(name="test_agent", model=model, tools=[])
|
|
62
|
+
result = await agent.run("Test prompt")
|
|
63
|
+
assert result.output.content == "Test response"
|
|
64
|
+
assert result.trace_id is not None
|
|
65
|
+
test_results.append(("Agent execution", True, ""))
|
|
66
|
+
tests_passed += 1
|
|
67
|
+
if verbose:
|
|
68
|
+
console.print(" [green]✓[/green] Agent executed successfully")
|
|
69
|
+
console.print(f" [dim] Output: {result.output.content}[/dim]")
|
|
70
|
+
console.print(f" [dim] Trace ID: {result.trace_id}[/dim]")
|
|
71
|
+
except Exception as e:
|
|
72
|
+
test_results.append(("Agent execution", False, str(e)))
|
|
73
|
+
tests_failed += 1
|
|
74
|
+
console.print(f" [red]✗[/red] Failed: {e}")
|
|
75
|
+
|
|
76
|
+
# Test 3: Tool execution
|
|
77
|
+
console.print("\n[cyan]Test 3: Tool execution[/cyan]")
|
|
78
|
+
try:
|
|
79
|
+
model = FakeModel(name="test")
|
|
80
|
+
model.add_response(
|
|
81
|
+
{
|
|
82
|
+
"content": "Calling echo",
|
|
83
|
+
"tool_calls": [
|
|
84
|
+
{"id": "call_1", "name": "echo", "arguments": {"message": "test"}}
|
|
85
|
+
],
|
|
86
|
+
}
|
|
87
|
+
)
|
|
88
|
+
model.add_response("Tool executed")
|
|
89
|
+
agent = Agent(name="test_agent", model=model, tools=[EchoTool()])
|
|
90
|
+
result = await agent.run("Use the echo tool")
|
|
91
|
+
test_results.append(("Tool execution", True, ""))
|
|
92
|
+
tests_passed += 1
|
|
93
|
+
if verbose:
|
|
94
|
+
console.print(" [green]✓[/green] Tool executed successfully")
|
|
95
|
+
console.print(f" [dim] Output: {result.output.content}[/dim]")
|
|
96
|
+
except Exception as e:
|
|
97
|
+
test_results.append(("Tool execution", False, str(e)))
|
|
98
|
+
tests_failed += 1
|
|
99
|
+
console.print(f" [red]✗[/red] Failed: {e}")
|
|
100
|
+
|
|
101
|
+
# Test 4: Tracing
|
|
102
|
+
console.print("\n[cyan]Test 4: Trace generation[/cyan]")
|
|
103
|
+
try:
|
|
104
|
+
trace_dir = Path(".routekit") / "traces" / "test"
|
|
105
|
+
model = FakeModel(name="test")
|
|
106
|
+
model.add_response("Traced response")
|
|
107
|
+
agent = Agent(name="test_agent", model=model, tools=[], trace_dir=trace_dir)
|
|
108
|
+
result = await agent.run("Test trace")
|
|
109
|
+
|
|
110
|
+
# Check if trace file exists
|
|
111
|
+
from routekitai.observability.exporters.jsonl import JSONLExporter
|
|
112
|
+
|
|
113
|
+
exporter = JSONLExporter(output_dir=trace_dir)
|
|
114
|
+
trace = await exporter.load(result.trace_id)
|
|
115
|
+
assert trace is not None
|
|
116
|
+
assert len(trace.events) > 0
|
|
117
|
+
test_results.append(("Trace generation", True, ""))
|
|
118
|
+
tests_passed += 1
|
|
119
|
+
if verbose:
|
|
120
|
+
console.print(" [green]✓[/green] Trace generated successfully")
|
|
121
|
+
console.print(f" [dim] Events: {len(trace.events)}[/dim]")
|
|
122
|
+
except Exception as e:
|
|
123
|
+
test_results.append(("Trace generation", False, str(e)))
|
|
124
|
+
tests_failed += 1
|
|
125
|
+
console.print(f" [red]✗[/red] Failed: {e}")
|
|
126
|
+
|
|
127
|
+
# Test 5: Replay
|
|
128
|
+
console.print("\n[cyan]Test 5: Trace replay[/cyan]")
|
|
129
|
+
try:
|
|
130
|
+
trace_dir = Path(".routekit") / "traces" / "test"
|
|
131
|
+
model = FakeModel(name="test")
|
|
132
|
+
model.add_response("Replay test")
|
|
133
|
+
agent = Agent(name="test_agent", model=model, tools=[], trace_dir=trace_dir)
|
|
134
|
+
result = await agent.run("Replay test")
|
|
135
|
+
trace_id = result.trace_id
|
|
136
|
+
|
|
137
|
+
# Replay
|
|
138
|
+
runtime = Runtime(trace_dir=trace_dir)
|
|
139
|
+
runtime.register_agent(agent)
|
|
140
|
+
replay_result = await runtime.replay(trace_id, "test_agent", verify_output=True)
|
|
141
|
+
assert replay_result.output.content == result.output.content
|
|
142
|
+
test_results.append(("Trace replay", True, ""))
|
|
143
|
+
tests_passed += 1
|
|
144
|
+
if verbose:
|
|
145
|
+
console.print(" [green]✓[/green] Replay successful")
|
|
146
|
+
console.print(
|
|
147
|
+
f" [dim] Output matches: {replay_result.output.content == result.output.content}[/dim]"
|
|
148
|
+
)
|
|
149
|
+
except Exception as e:
|
|
150
|
+
test_results.append(("Trace replay", False, str(e)))
|
|
151
|
+
tests_failed += 1
|
|
152
|
+
console.print(f" [red]✗[/red] Failed: {e}")
|
|
153
|
+
|
|
154
|
+
# Summary
|
|
155
|
+
console.print("\n" + "=" * 50)
|
|
156
|
+
table = Table(title="Test Results", show_header=True, header_style="bold magenta")
|
|
157
|
+
table.add_column("Test", style="cyan")
|
|
158
|
+
table.add_column("Status", style="green")
|
|
159
|
+
table.add_column("Error", style="red")
|
|
160
|
+
|
|
161
|
+
for test_name, passed, error in test_results:
|
|
162
|
+
status = "[green]PASS[/green]" if passed else "[red]FAIL[/red]"
|
|
163
|
+
table.add_row(test_name, status, error or "-")
|
|
164
|
+
|
|
165
|
+
console.print(table)
|
|
166
|
+
console.print(f"\n[bold]Total: {tests_passed} passed, {tests_failed} failed[/bold]")
|
|
167
|
+
|
|
168
|
+
if tests_failed > 0:
|
|
169
|
+
console.print("\n[red]Some tests failed. Check the output above for details.[/red]")
|
|
170
|
+
raise typer.Exit(1)
|
|
171
|
+
else:
|
|
172
|
+
console.print("\n[green]All tests passed! ✓[/green]")
|
|
173
|
+
|
|
174
|
+
asyncio.run(_run_tests())
|
|
175
|
+
|
|
176
|
+
|
|
177
|
+
if __name__ == "__main__" and app is not None:
|
|
178
|
+
app()
|
routekitai/cli/trace.py
ADDED
|
@@ -0,0 +1,209 @@
|
|
|
1
|
+
"""CLI command for viewing traces."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
import json
|
|
5
|
+
from pathlib import Path
|
|
6
|
+
|
|
7
|
+
try:
|
|
8
|
+
import typer
|
|
9
|
+
from rich.console import Console
|
|
10
|
+
from rich.json import JSON
|
|
11
|
+
from rich.table import Table
|
|
12
|
+
except ImportError as e:
|
|
13
|
+
raise ImportError("CLI dependencies not installed. Install with: pip install typer rich") from e
|
|
14
|
+
|
|
15
|
+
from routekitai.observability.analyzer import TraceAnalyzer
|
|
16
|
+
from routekitai.observability.exporters.jsonl import JSONLExporter
|
|
17
|
+
from routekitai.observability.trace import Trace
|
|
18
|
+
|
|
19
|
+
app = typer.Typer(name="trace", help="View and inspect agent execution traces")
|
|
20
|
+
console = Console()
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def trace_command(
|
|
24
|
+
trace_id: str = typer.Argument(..., help="Trace ID to view"),
|
|
25
|
+
trace_dir: str | None = typer.Option(
|
|
26
|
+
None, "--trace-dir", "-t", help="Directory containing trace files"
|
|
27
|
+
),
|
|
28
|
+
format: str = typer.Option(
|
|
29
|
+
"table", "--format", "-f", help="Output format: table, json, raw, timeline, steps"
|
|
30
|
+
),
|
|
31
|
+
) -> None:
|
|
32
|
+
"""View an agent execution trace.
|
|
33
|
+
|
|
34
|
+
Examples:
|
|
35
|
+
routkitai trace abc123
|
|
36
|
+
routkitai trace abc123 --format json
|
|
37
|
+
routkitai trace abc123 --trace-dir ./custom_traces
|
|
38
|
+
"""
|
|
39
|
+
# Determine trace directory
|
|
40
|
+
trace_path: Path
|
|
41
|
+
if trace_dir is None:
|
|
42
|
+
trace_path = Path(".routekit") / "traces"
|
|
43
|
+
else:
|
|
44
|
+
trace_path = Path(trace_dir)
|
|
45
|
+
|
|
46
|
+
if not trace_path.exists():
|
|
47
|
+
console.print(f"[red]Error: Trace directory not found: {trace_dir}[/red]")
|
|
48
|
+
raise typer.Exit(1)
|
|
49
|
+
|
|
50
|
+
# Load trace
|
|
51
|
+
exporter = JSONLExporter(output_dir=trace_path)
|
|
52
|
+
trace = asyncio.run(exporter.load(trace_id))
|
|
53
|
+
|
|
54
|
+
if trace is None:
|
|
55
|
+
console.print(f"[red]Error: Trace '{trace_id}' not found in {trace_path}[/red]")
|
|
56
|
+
raise typer.Exit(1)
|
|
57
|
+
|
|
58
|
+
analyzer = TraceAnalyzer()
|
|
59
|
+
|
|
60
|
+
if format == "json":
|
|
61
|
+
# Output as JSON
|
|
62
|
+
trace_data = {
|
|
63
|
+
"trace_id": trace.trace_id,
|
|
64
|
+
"metadata": trace.metadata,
|
|
65
|
+
"events": [event.model_dump() for event in trace.events],
|
|
66
|
+
}
|
|
67
|
+
console.print(JSON(json.dumps(trace_data, indent=2)))
|
|
68
|
+
elif format == "raw":
|
|
69
|
+
# Raw JSONL output
|
|
70
|
+
trace_file = trace_path / f"{trace_id}.jsonl"
|
|
71
|
+
if trace_file.exists():
|
|
72
|
+
console.print(trace_file.read_text())
|
|
73
|
+
else:
|
|
74
|
+
console.print(f"[red]Error: Trace file not found: {trace_file}[/red]")
|
|
75
|
+
elif format == "timeline":
|
|
76
|
+
# Timeline visualization
|
|
77
|
+
_display_timeline(trace, analyzer)
|
|
78
|
+
elif format == "steps":
|
|
79
|
+
# Step-by-step execution view
|
|
80
|
+
_display_steps(trace, analyzer)
|
|
81
|
+
else:
|
|
82
|
+
# Table format (default)
|
|
83
|
+
console.print(f"\n[bold]Trace: {trace.trace_id}[/bold]")
|
|
84
|
+
if trace.metadata:
|
|
85
|
+
console.print(f"[dim]Metadata: {json.dumps(trace.metadata, indent=2)}[/dim]\n")
|
|
86
|
+
|
|
87
|
+
table = Table(title="Trace Events", show_header=True, header_style="bold magenta")
|
|
88
|
+
table.add_column("Type", style="cyan")
|
|
89
|
+
table.add_column("Timestamp", style="green")
|
|
90
|
+
table.add_column("Data", style="yellow")
|
|
91
|
+
|
|
92
|
+
for event in trace.events:
|
|
93
|
+
data_str = json.dumps(event.data, indent=2) if event.data else ""
|
|
94
|
+
# Truncate long data
|
|
95
|
+
if len(data_str) > 200:
|
|
96
|
+
data_str = data_str[:200] + "..."
|
|
97
|
+
table.add_row(event.type, f"{event.timestamp:.3f}", data_str)
|
|
98
|
+
|
|
99
|
+
console.print(table)
|
|
100
|
+
console.print(f"\n[dim]Total events: {len(trace.events)}[/dim]")
|
|
101
|
+
|
|
102
|
+
# Show quick metrics
|
|
103
|
+
metrics = analyzer.analyze(trace)
|
|
104
|
+
console.print(
|
|
105
|
+
f"\n[dim]Duration: {metrics.total_duration_ms:.2f} ms | "
|
|
106
|
+
f"Model calls: {metrics.model_calls} | "
|
|
107
|
+
f"Tool calls: {metrics.tool_calls} | "
|
|
108
|
+
f"Errors: {metrics.errors}[/dim]"
|
|
109
|
+
)
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
def _display_timeline(trace: Trace, analyzer: TraceAnalyzer) -> None:
|
|
113
|
+
"""Display trace as a timeline.
|
|
114
|
+
|
|
115
|
+
Args:
|
|
116
|
+
trace: Trace to display
|
|
117
|
+
analyzer: Trace analyzer instance
|
|
118
|
+
"""
|
|
119
|
+
console.print(f"\n[bold cyan]Timeline: {trace.trace_id}[/bold cyan]\n")
|
|
120
|
+
|
|
121
|
+
timeline = analyzer.get_timeline(trace)
|
|
122
|
+
if not timeline:
|
|
123
|
+
console.print("[yellow]No events in trace[/yellow]")
|
|
124
|
+
return
|
|
125
|
+
|
|
126
|
+
# Create timeline visualization
|
|
127
|
+
max_time = max(entry["relative_time_ms"] for entry in timeline) if timeline else 0
|
|
128
|
+
bar_width = 60
|
|
129
|
+
|
|
130
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
131
|
+
table.add_column("Time (ms)", style="green", width=12)
|
|
132
|
+
table.add_column("Event Type", style="cyan", width=20)
|
|
133
|
+
table.add_column("Duration (ms)", style="yellow", width=15)
|
|
134
|
+
table.add_column("Timeline", style="white", width=bar_width)
|
|
135
|
+
table.add_column("Details", style="white")
|
|
136
|
+
|
|
137
|
+
for entry in timeline:
|
|
138
|
+
event = entry["event"]
|
|
139
|
+
time_ms = entry["relative_time_ms"]
|
|
140
|
+
duration_ms = entry["duration_ms"]
|
|
141
|
+
|
|
142
|
+
# Create visual bar
|
|
143
|
+
if max_time > 0:
|
|
144
|
+
bar_pos = int((time_ms / max_time) * bar_width)
|
|
145
|
+
bar = " " * bar_pos + "█"
|
|
146
|
+
else:
|
|
147
|
+
bar = "█"
|
|
148
|
+
|
|
149
|
+
duration_str = f"{duration_ms:.2f}" if duration_ms > 0 else "-"
|
|
150
|
+
details = json.dumps(event.data, indent=2)[:100]
|
|
151
|
+
if len(json.dumps(event.data)) > 100:
|
|
152
|
+
details += "..."
|
|
153
|
+
|
|
154
|
+
table.add_row(
|
|
155
|
+
f"{time_ms:.2f}",
|
|
156
|
+
event.type,
|
|
157
|
+
duration_str,
|
|
158
|
+
bar,
|
|
159
|
+
details,
|
|
160
|
+
)
|
|
161
|
+
|
|
162
|
+
console.print(table)
|
|
163
|
+
|
|
164
|
+
|
|
165
|
+
def _display_steps(trace: Trace, analyzer: TraceAnalyzer) -> None:
|
|
166
|
+
"""Display step-by-step execution view.
|
|
167
|
+
|
|
168
|
+
Args:
|
|
169
|
+
trace: Trace to display
|
|
170
|
+
analyzer: Trace analyzer instance
|
|
171
|
+
"""
|
|
172
|
+
console.print(f"\n[bold cyan]Step-by-Step Execution: {trace.trace_id}[/bold cyan]\n")
|
|
173
|
+
|
|
174
|
+
steps = analyzer.get_step_sequence(trace)
|
|
175
|
+
if not steps:
|
|
176
|
+
console.print("[yellow]No steps found in trace[/yellow]")
|
|
177
|
+
return
|
|
178
|
+
|
|
179
|
+
for i, step in enumerate(steps, 1):
|
|
180
|
+
step_id = step["step_id"]
|
|
181
|
+
step_type = step["step_type"]
|
|
182
|
+
duration_ms = step["duration_ms"]
|
|
183
|
+
error = step.get("error")
|
|
184
|
+
|
|
185
|
+
# Step header
|
|
186
|
+
status = "[red]✗ ERROR[/red]" if error else "[green]✓ OK[/green]"
|
|
187
|
+
console.print(f"\n[bold]Step {i}: {step_id}[/bold] ({step_type}) {status}")
|
|
188
|
+
console.print(f"[dim]Duration: {duration_ms:.2f} ms[/dim]")
|
|
189
|
+
|
|
190
|
+
if error:
|
|
191
|
+
console.print(f"[red]Error: {error}[/red]")
|
|
192
|
+
|
|
193
|
+
# Step events
|
|
194
|
+
if step["events"]:
|
|
195
|
+
event_table = Table(show_header=True, header_style="bold")
|
|
196
|
+
event_table.add_column("Event", style="cyan", width=20)
|
|
197
|
+
event_table.add_column("Data", style="white")
|
|
198
|
+
|
|
199
|
+
for event in step["events"]:
|
|
200
|
+
data_str = json.dumps(event.data, indent=2)[:200]
|
|
201
|
+
if len(json.dumps(event.data)) > 200:
|
|
202
|
+
data_str += "..."
|
|
203
|
+
event_table.add_row(event.type, data_str)
|
|
204
|
+
|
|
205
|
+
console.print(event_table)
|
|
206
|
+
|
|
207
|
+
|
|
208
|
+
if __name__ == "__main__" and app is not None:
|
|
209
|
+
app()
|
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
"""CLI command for trace analysis and metrics."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import typer
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
except ImportError as e:
|
|
11
|
+
raise ImportError("CLI dependencies not installed. Install with: pip install typer rich") from e
|
|
12
|
+
|
|
13
|
+
from routekitai.observability.analyzer import TraceAnalyzer
|
|
14
|
+
from routekitai.observability.exporters.jsonl import JSONLExporter
|
|
15
|
+
|
|
16
|
+
app = typer.Typer(name="trace-analyze", help="Analyze trace metrics and performance")
|
|
17
|
+
console = Console()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def analyze_command(
|
|
21
|
+
trace_id: str = typer.Argument(..., help="Trace ID to analyze"),
|
|
22
|
+
trace_dir: str | None = typer.Option(
|
|
23
|
+
None, "--trace-dir", "-t", help="Directory containing trace files"
|
|
24
|
+
),
|
|
25
|
+
) -> None:
|
|
26
|
+
"""Analyze a trace and display metrics.
|
|
27
|
+
|
|
28
|
+
Examples:
|
|
29
|
+
routkitai trace-analyze abc123
|
|
30
|
+
routkitai trace-analyze abc123 --trace-dir ./custom_traces
|
|
31
|
+
"""
|
|
32
|
+
# Determine trace directory
|
|
33
|
+
trace_path: Path
|
|
34
|
+
if trace_dir is None:
|
|
35
|
+
trace_path = Path(".routekit") / "traces"
|
|
36
|
+
else:
|
|
37
|
+
trace_path = Path(trace_dir)
|
|
38
|
+
|
|
39
|
+
if not trace_path.exists():
|
|
40
|
+
console.print(f"[red]Error: Trace directory not found: {trace_path}[/red]")
|
|
41
|
+
raise typer.Exit(1)
|
|
42
|
+
|
|
43
|
+
# Load trace
|
|
44
|
+
exporter = JSONLExporter(output_dir=trace_path)
|
|
45
|
+
trace = asyncio.run(exporter.load(trace_id))
|
|
46
|
+
|
|
47
|
+
if trace is None:
|
|
48
|
+
console.print(f"[red]Error: Trace '{trace_id}' not found in {trace_path}[/red]")
|
|
49
|
+
raise typer.Exit(1)
|
|
50
|
+
|
|
51
|
+
# Analyze trace
|
|
52
|
+
analyzer = TraceAnalyzer()
|
|
53
|
+
metrics = analyzer.analyze(trace)
|
|
54
|
+
|
|
55
|
+
# Display metrics
|
|
56
|
+
console.print(f"\n[bold cyan]Trace Analysis: {trace_id}[/bold cyan]\n")
|
|
57
|
+
|
|
58
|
+
# Overview table
|
|
59
|
+
overview_table = Table(title="Overview", show_header=True, header_style="bold magenta")
|
|
60
|
+
overview_table.add_column("Metric", style="cyan")
|
|
61
|
+
overview_table.add_column("Value", style="green")
|
|
62
|
+
|
|
63
|
+
overview_table.add_row("Total Events", str(metrics.total_events))
|
|
64
|
+
overview_table.add_row("Total Duration", f"{metrics.total_duration_ms:.2f} ms")
|
|
65
|
+
overview_table.add_row("Steps", str(metrics.steps))
|
|
66
|
+
overview_table.add_row("Errors", str(metrics.errors))
|
|
67
|
+
overview_table.add_row("Error Rate", f"{metrics.error_rate * 100:.2f}%")
|
|
68
|
+
|
|
69
|
+
console.print(overview_table)
|
|
70
|
+
|
|
71
|
+
# Model calls table
|
|
72
|
+
model_table = Table(title="Model Calls", show_header=True, header_style="bold magenta")
|
|
73
|
+
model_table.add_column("Metric", style="cyan")
|
|
74
|
+
model_table.add_column("Value", style="green")
|
|
75
|
+
|
|
76
|
+
model_table.add_row("Total Calls", str(metrics.model_calls))
|
|
77
|
+
model_table.add_row("Avg Latency", f"{metrics.avg_model_latency_ms:.2f} ms")
|
|
78
|
+
model_table.add_row("Total Tokens", str(metrics.total_tokens))
|
|
79
|
+
model_table.add_row("Prompt Tokens", str(metrics.prompt_tokens))
|
|
80
|
+
model_table.add_row("Completion Tokens", str(metrics.completion_tokens))
|
|
81
|
+
|
|
82
|
+
console.print("\n")
|
|
83
|
+
console.print(model_table)
|
|
84
|
+
|
|
85
|
+
# Tool calls table
|
|
86
|
+
tool_table = Table(title="Tool Calls", show_header=True, header_style="bold magenta")
|
|
87
|
+
tool_table.add_column("Metric", style="cyan")
|
|
88
|
+
tool_table.add_column("Value", style="green")
|
|
89
|
+
|
|
90
|
+
tool_table.add_row("Total Calls", str(metrics.tool_calls))
|
|
91
|
+
tool_table.add_row("Avg Latency", f"{metrics.avg_tool_latency_ms:.2f} ms")
|
|
92
|
+
|
|
93
|
+
console.print("\n")
|
|
94
|
+
console.print(tool_table)
|
|
95
|
+
|
|
96
|
+
# Cost estimation (if tokens available)
|
|
97
|
+
if metrics.total_tokens > 0:
|
|
98
|
+
console.print("\n[bold yellow]Cost Estimation (approximate):[/bold yellow]")
|
|
99
|
+
cost_table = Table(show_header=True, header_style="bold magenta")
|
|
100
|
+
cost_table.add_column("Provider", style="cyan")
|
|
101
|
+
cost_table.add_column("Model", style="yellow")
|
|
102
|
+
cost_table.add_column("Estimated Cost", style="green")
|
|
103
|
+
|
|
104
|
+
# OpenAI GPT-4 pricing (example)
|
|
105
|
+
gpt4_prompt_cost = (metrics.prompt_tokens / 1000) * 0.03
|
|
106
|
+
gpt4_completion_cost = (metrics.completion_tokens / 1000) * 0.06
|
|
107
|
+
cost_table.add_row("OpenAI", "gpt-4", f"${gpt4_prompt_cost + gpt4_completion_cost:.4f}")
|
|
108
|
+
|
|
109
|
+
# OpenAI GPT-3.5 pricing
|
|
110
|
+
gpt35_prompt_cost = (metrics.prompt_tokens / 1000) * 0.0015
|
|
111
|
+
gpt35_completion_cost = (metrics.completion_tokens / 1000) * 0.002
|
|
112
|
+
cost_table.add_row(
|
|
113
|
+
"OpenAI", "gpt-3.5-turbo", f"${gpt35_prompt_cost + gpt35_completion_cost:.4f}"
|
|
114
|
+
)
|
|
115
|
+
|
|
116
|
+
console.print(cost_table)
|
|
117
|
+
|
|
118
|
+
|
|
119
|
+
if __name__ == "__main__" and app is not None:
|
|
120
|
+
app()
|
|
@@ -0,0 +1,126 @@
|
|
|
1
|
+
"""CLI command for searching traces."""
|
|
2
|
+
|
|
3
|
+
import asyncio
|
|
4
|
+
from pathlib import Path
|
|
5
|
+
|
|
6
|
+
try:
|
|
7
|
+
import typer
|
|
8
|
+
from rich.console import Console
|
|
9
|
+
from rich.table import Table
|
|
10
|
+
except ImportError as e:
|
|
11
|
+
raise ImportError("CLI dependencies not installed. Install with: pip install typer rich") from e
|
|
12
|
+
|
|
13
|
+
from routekitai.observability.analyzer import TraceAnalyzer
|
|
14
|
+
from routekitai.observability.exporters.jsonl import JSONLExporter
|
|
15
|
+
|
|
16
|
+
app = typer.Typer(name="trace-search", help="Search traces by content")
|
|
17
|
+
console = Console()
|
|
18
|
+
|
|
19
|
+
|
|
20
|
+
def search_command(
|
|
21
|
+
query: str = typer.Argument(..., help="Search query"),
|
|
22
|
+
trace_id: str | None = typer.Option(
|
|
23
|
+
None, "--trace-id", "-t", help="Specific trace ID to search (optional)"
|
|
24
|
+
),
|
|
25
|
+
trace_dir: str | None = typer.Option(
|
|
26
|
+
None, "--trace-dir", "-d", help="Directory containing trace files"
|
|
27
|
+
),
|
|
28
|
+
event_type: str | None = typer.Option(None, "--event-type", "-e", help="Filter by event type"),
|
|
29
|
+
) -> None:
|
|
30
|
+
"""Search traces by content.
|
|
31
|
+
|
|
32
|
+
Examples:
|
|
33
|
+
routkitai trace-search "error"
|
|
34
|
+
routkitai trace-search "model" --trace-id abc123
|
|
35
|
+
routkitai trace-search "tool" --event-type tool_called
|
|
36
|
+
"""
|
|
37
|
+
# Determine trace directory
|
|
38
|
+
trace_path: Path
|
|
39
|
+
if trace_dir is None:
|
|
40
|
+
trace_path = Path(".routekit") / "traces"
|
|
41
|
+
else:
|
|
42
|
+
trace_path = Path(trace_dir)
|
|
43
|
+
|
|
44
|
+
if not trace_path.exists():
|
|
45
|
+
console.print(f"[red]Error: Trace directory not found: {trace_path}[/red]")
|
|
46
|
+
raise typer.Exit(1)
|
|
47
|
+
|
|
48
|
+
exporter = JSONLExporter(output_dir=trace_path)
|
|
49
|
+
analyzer = TraceAnalyzer()
|
|
50
|
+
|
|
51
|
+
# Search in specific trace or all traces
|
|
52
|
+
if trace_id:
|
|
53
|
+
trace = asyncio.run(exporter.load(trace_id))
|
|
54
|
+
if trace is None:
|
|
55
|
+
console.print(f"[red]Error: Trace '{trace_id}' not found[/red]")
|
|
56
|
+
raise typer.Exit(1)
|
|
57
|
+
|
|
58
|
+
results = analyzer.search(trace, query)
|
|
59
|
+
if event_type:
|
|
60
|
+
results = [r for r in results if r.type == event_type]
|
|
61
|
+
|
|
62
|
+
_display_search_results(trace_id, results)
|
|
63
|
+
else:
|
|
64
|
+
# Search all traces
|
|
65
|
+
trace_files = list(trace_path.glob("*.jsonl"))
|
|
66
|
+
if not trace_files:
|
|
67
|
+
console.print(f"[yellow]No traces found in {trace_path}[/yellow]")
|
|
68
|
+
raise typer.Exit(0)
|
|
69
|
+
|
|
70
|
+
all_results: list[tuple[str, list]] = []
|
|
71
|
+
for trace_file in trace_files:
|
|
72
|
+
trace_id_from_file = trace_file.stem
|
|
73
|
+
trace = asyncio.run(exporter.load(trace_id_from_file))
|
|
74
|
+
if trace:
|
|
75
|
+
results = analyzer.search(trace, query)
|
|
76
|
+
if event_type:
|
|
77
|
+
results = [r for r in results if r.type == event_type]
|
|
78
|
+
if results:
|
|
79
|
+
all_results.append((trace_id_from_file, results))
|
|
80
|
+
|
|
81
|
+
if not all_results:
|
|
82
|
+
console.print(f"[yellow]No matches found for query: '{query}'[/yellow]")
|
|
83
|
+
raise typer.Exit(0)
|
|
84
|
+
|
|
85
|
+
console.print(f"\n[bold]Search Results for '{query}':[/bold]\n")
|
|
86
|
+
for tid, results in all_results:
|
|
87
|
+
console.print(f"[cyan]Trace: {tid}[/cyan] ({len(results)} matches)")
|
|
88
|
+
_display_search_results(tid, results, show_trace_id=False)
|
|
89
|
+
console.print()
|
|
90
|
+
|
|
91
|
+
|
|
92
|
+
def _display_search_results(trace_id: str, results: list, show_trace_id: bool = True) -> None:
|
|
93
|
+
"""Display search results in a table.
|
|
94
|
+
|
|
95
|
+
Args:
|
|
96
|
+
trace_id: Trace ID
|
|
97
|
+
results: List of matching events
|
|
98
|
+
show_trace_id: Whether to show trace ID in table
|
|
99
|
+
"""
|
|
100
|
+
if not results:
|
|
101
|
+
console.print("[yellow]No matches found[/yellow]")
|
|
102
|
+
return
|
|
103
|
+
|
|
104
|
+
table = Table(show_header=True, header_style="bold magenta")
|
|
105
|
+
if show_trace_id:
|
|
106
|
+
table.add_column("Trace ID", style="cyan")
|
|
107
|
+
table.add_column("Event Type", style="yellow")
|
|
108
|
+
table.add_column("Timestamp", style="green")
|
|
109
|
+
table.add_column("Preview", style="white")
|
|
110
|
+
|
|
111
|
+
for event in results[:50]: # Limit to 50 results
|
|
112
|
+
preview = str(event.data)[:100]
|
|
113
|
+
if len(str(event.data)) > 100:
|
|
114
|
+
preview += "..."
|
|
115
|
+
row = [event.type, f"{event.timestamp:.3f}", preview]
|
|
116
|
+
if show_trace_id:
|
|
117
|
+
row.insert(0, trace_id)
|
|
118
|
+
table.add_row(*row)
|
|
119
|
+
|
|
120
|
+
console.print(table)
|
|
121
|
+
if len(results) > 50:
|
|
122
|
+
console.print(f"[dim]... and {len(results) - 50} more results[/dim]")
|
|
123
|
+
|
|
124
|
+
|
|
125
|
+
if __name__ == "__main__" and app is not None:
|
|
126
|
+
app()
|
|
@@ -0,0 +1,58 @@
|
|
|
1
|
+
"""routkitai core primitives."""
|
|
2
|
+
|
|
3
|
+
from routekitai.core.agent import Agent, RunResult
|
|
4
|
+
from routekitai.core.errors import (
|
|
5
|
+
ModelError,
|
|
6
|
+
PolicyError,
|
|
7
|
+
RouteKitError,
|
|
8
|
+
RuntimeError,
|
|
9
|
+
ToolError,
|
|
10
|
+
)
|
|
11
|
+
from routekitai.core.message import Message, MessageRole
|
|
12
|
+
from routekitai.core.model import Model, ModelResponse, StreamEvent, ToolCall, Usage
|
|
13
|
+
from routekitai.core.policies import (
|
|
14
|
+
FunctionCallingPolicy,
|
|
15
|
+
GraphPolicy,
|
|
16
|
+
PlanExecutePolicy,
|
|
17
|
+
ReActPolicy,
|
|
18
|
+
SupervisorPolicy,
|
|
19
|
+
)
|
|
20
|
+
from routekitai.core.policy import Action, Final, ModelAction, Parallel, Policy, ToolAction
|
|
21
|
+
from routekitai.core.policy_adapter import PolicyAdapter
|
|
22
|
+
from routekitai.core.runtime import Runtime
|
|
23
|
+
from routekitai.core.tool import Tool
|
|
24
|
+
from routekitai.core.tools import EchoTool, FileReadTool, HttpGetTool
|
|
25
|
+
|
|
26
|
+
__all__ = [
|
|
27
|
+
"Action",
|
|
28
|
+
"Agent",
|
|
29
|
+
"Final",
|
|
30
|
+
"FunctionCallingPolicy",
|
|
31
|
+
"GraphPolicy",
|
|
32
|
+
"Message",
|
|
33
|
+
"MessageRole",
|
|
34
|
+
"Model",
|
|
35
|
+
"ModelAction",
|
|
36
|
+
"ModelResponse",
|
|
37
|
+
"Parallel",
|
|
38
|
+
"PlanExecutePolicy",
|
|
39
|
+
"Policy",
|
|
40
|
+
"PolicyAdapter",
|
|
41
|
+
"PolicyError",
|
|
42
|
+
"ReActPolicy",
|
|
43
|
+
"RouteKitError",
|
|
44
|
+
"ModelError",
|
|
45
|
+
"ToolError",
|
|
46
|
+
"RuntimeError",
|
|
47
|
+
"RunResult",
|
|
48
|
+
"Runtime",
|
|
49
|
+
"StreamEvent",
|
|
50
|
+
"SupervisorPolicy",
|
|
51
|
+
"Tool",
|
|
52
|
+
"ToolAction",
|
|
53
|
+
"ToolCall",
|
|
54
|
+
"Usage",
|
|
55
|
+
"EchoTool",
|
|
56
|
+
"HttpGetTool",
|
|
57
|
+
"FileReadTool",
|
|
58
|
+
]
|