prela 0.1.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (71) hide show
  1. prela/__init__.py +394 -0
  2. prela/_version.py +3 -0
  3. prela/contrib/CLI.md +431 -0
  4. prela/contrib/README.md +118 -0
  5. prela/contrib/__init__.py +5 -0
  6. prela/contrib/cli.py +1063 -0
  7. prela/contrib/explorer.py +571 -0
  8. prela/core/__init__.py +64 -0
  9. prela/core/clock.py +98 -0
  10. prela/core/context.py +228 -0
  11. prela/core/replay.py +403 -0
  12. prela/core/sampler.py +178 -0
  13. prela/core/span.py +295 -0
  14. prela/core/tracer.py +498 -0
  15. prela/evals/__init__.py +94 -0
  16. prela/evals/assertions/README.md +484 -0
  17. prela/evals/assertions/__init__.py +78 -0
  18. prela/evals/assertions/base.py +90 -0
  19. prela/evals/assertions/multi_agent.py +625 -0
  20. prela/evals/assertions/semantic.py +223 -0
  21. prela/evals/assertions/structural.py +443 -0
  22. prela/evals/assertions/tool.py +380 -0
  23. prela/evals/case.py +370 -0
  24. prela/evals/n8n/__init__.py +69 -0
  25. prela/evals/n8n/assertions.py +450 -0
  26. prela/evals/n8n/runner.py +497 -0
  27. prela/evals/reporters/README.md +184 -0
  28. prela/evals/reporters/__init__.py +32 -0
  29. prela/evals/reporters/console.py +251 -0
  30. prela/evals/reporters/json.py +176 -0
  31. prela/evals/reporters/junit.py +278 -0
  32. prela/evals/runner.py +525 -0
  33. prela/evals/suite.py +316 -0
  34. prela/exporters/__init__.py +27 -0
  35. prela/exporters/base.py +189 -0
  36. prela/exporters/console.py +443 -0
  37. prela/exporters/file.py +322 -0
  38. prela/exporters/http.py +394 -0
  39. prela/exporters/multi.py +154 -0
  40. prela/exporters/otlp.py +388 -0
  41. prela/instrumentation/ANTHROPIC.md +297 -0
  42. prela/instrumentation/LANGCHAIN.md +480 -0
  43. prela/instrumentation/OPENAI.md +59 -0
  44. prela/instrumentation/__init__.py +49 -0
  45. prela/instrumentation/anthropic.py +1436 -0
  46. prela/instrumentation/auto.py +129 -0
  47. prela/instrumentation/base.py +436 -0
  48. prela/instrumentation/langchain.py +959 -0
  49. prela/instrumentation/llamaindex.py +719 -0
  50. prela/instrumentation/multi_agent/__init__.py +48 -0
  51. prela/instrumentation/multi_agent/autogen.py +357 -0
  52. prela/instrumentation/multi_agent/crewai.py +404 -0
  53. prela/instrumentation/multi_agent/langgraph.py +299 -0
  54. prela/instrumentation/multi_agent/models.py +203 -0
  55. prela/instrumentation/multi_agent/swarm.py +231 -0
  56. prela/instrumentation/n8n/__init__.py +68 -0
  57. prela/instrumentation/n8n/code_node.py +534 -0
  58. prela/instrumentation/n8n/models.py +336 -0
  59. prela/instrumentation/n8n/webhook.py +489 -0
  60. prela/instrumentation/openai.py +1198 -0
  61. prela/license.py +245 -0
  62. prela/replay/__init__.py +31 -0
  63. prela/replay/comparison.py +390 -0
  64. prela/replay/engine.py +1227 -0
  65. prela/replay/loader.py +231 -0
  66. prela/replay/result.py +196 -0
  67. prela-0.1.0.dist-info/METADATA +399 -0
  68. prela-0.1.0.dist-info/RECORD +71 -0
  69. prela-0.1.0.dist-info/WHEEL +4 -0
  70. prela-0.1.0.dist-info/entry_points.txt +2 -0
  71. prela-0.1.0.dist-info/licenses/LICENSE +190 -0
prela/contrib/CLI.md ADDED
@@ -0,0 +1,431 @@
1
+ # Prela CLI Documentation
2
+
3
+ The Prela CLI provides command-line tools for managing Prela configuration, viewing traces, searching traces, and running evaluations.
4
+
5
+ ## Installation
6
+
7
+ Install the CLI dependencies:
8
+
9
+ ```bash
10
+ pip install prela[cli]
11
+ ```
12
+
13
+ This installs:
14
+ - `typer` - CLI framework
15
+ - `pyyaml` - YAML configuration support
16
+ - `rich` - Beautiful terminal output
17
+
18
+ ## Commands
19
+
20
+ ### `prela init`
21
+
22
+ Initialize Prela configuration with interactive prompts.
23
+
24
+ **Usage:**
25
+ ```bash
26
+ prela init
27
+ ```
28
+
29
+ **Interactive Prompts:**
30
+ 1. Service name (default: `my-agent`)
31
+ 2. Exporter type (`console` or `file`, default: `file`)
32
+ 3. Trace directory (default: `./traces`, only for file exporter)
33
+ 4. Sample rate (0.0-1.0, default: `1.0`)
34
+
35
+ **Output:**
36
+ - Creates `.prela.yaml` configuration file in current directory
37
+ - Creates trace directory (if using file exporter)
38
+
39
+ **Example:**
40
+ ```bash
41
+ $ prela init
42
+ Prela Configuration Setup
43
+
44
+ Service name [my-agent]: my-chatbot
45
+ Exporter (console/file) [file]: file
46
+ Trace directory [./traces]: ./my-traces
47
+ Sample rate (0.0-1.0) [1.0]: 0.5
48
+
49
+ ✓ Configuration saved to .prela.yaml
50
+ ✓ Created trace directory: ./my-traces
51
+ ```
52
+
53
+ **Configuration File Format:**
54
+ ```yaml
55
+ service_name: my-chatbot
56
+ exporter: file
57
+ trace_dir: ./my-traces
58
+ sample_rate: 0.5
59
+ ```
60
+
61
+ ---
62
+
63
+ ### `prela trace list`
64
+
65
+ List recent traces from file exporter.
66
+
67
+ **Usage:**
68
+ ```bash
69
+ prela trace list [OPTIONS]
70
+ ```
71
+
72
+ **Options:**
73
+ - `--limit, -n INTEGER` - Maximum number of traces to show (default: 20)
74
+ - `--since, -s DURATION` - Show traces since duration ago (e.g., `1h`, `30m`, `2d`)
75
+
76
+ **Duration Format:**
77
+ - `s` - seconds (e.g., `30s`)
78
+ - `m` - minutes (e.g., `15m`)
79
+ - `h` - hours (e.g., `2h`)
80
+ - `d` - days (e.g., `3d`)
81
+
82
+ **Output:**
83
+ Displays table with:
84
+ - Trace ID (first 16 chars)
85
+ - Root span name
86
+ - Duration (ms or seconds)
87
+ - Status (success/error/pending)
88
+ - Number of spans
89
+ - Timestamp
90
+
91
+ **Example:**
92
+ ```bash
93
+ $ prela trace list --limit 10 --since 1h
94
+
95
+ ┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓
96
+ ┃ Trace ID ┃ Root Span ┃ Duration ┃ Status ┃ Spans ┃ Time ┃
97
+ ┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩
98
+ │ trace-abc123... │ anthropic.message │ 1.23s │ ✓ │ 3 │ 2025-01-26 10:30:45 │
99
+ │ trace-def456... │ langchain.agent │ 2.45s │ ✓ │ 8 │ 2025-01-26 10:28:12 │
100
+ │ trace-ghi789... │ custom_operation │ 567ms │ ✗ │ 2 │ 2025-01-26 10:25:33 │
101
+ └──────────────────┴────────────────────┴──────────┴────────┴───────┴─────────────────────┘
102
+ ```
103
+
104
+ ---
105
+
106
+ ### `prela trace show`
107
+
108
+ Display full trace tree with all spans, attributes, and events.
109
+
110
+ **Usage:**
111
+ ```bash
112
+ prela trace show TRACE_ID
113
+ ```
114
+
115
+ **Arguments:**
116
+ - `TRACE_ID` - Full or partial trace ID (supports prefix matching)
117
+
118
+ **Output:**
119
+ 1. Trace tree visualization (hierarchical spans)
120
+ 2. Detailed span information:
121
+ - Span ID
122
+ - Type (agent/llm/tool/etc.)
123
+ - Status
124
+ - Attributes (key-value pairs)
125
+ - Events (timestamped events)
126
+
127
+ **Example:**
128
+ ```bash
129
+ $ prela trace show trace-abc123
130
+
131
+ Trace: trace-abc123456789
132
+
133
+ anthropic.messages.create (llm) success 1.23s
134
+ └── anthropic.stream.process (custom) success 450ms
135
+
136
+ Span Details:
137
+
138
+ anthropic.messages.create
139
+ Span ID: span-001
140
+ Type: llm
141
+ Status: success
142
+ Attributes:
143
+ llm.vendor: anthropic
144
+ llm.model: claude-3-5-sonnet-20241022
145
+ llm.input_tokens: 150
146
+ llm.output_tokens: 89
147
+ llm.temperature: 1.0
148
+ Events (2):
149
+ - llm.request @ 2025-01-26T10:30:45.123456Z
150
+ - llm.response @ 2025-01-26T10:30:46.356789Z
151
+
152
+ anthropic.stream.process
153
+ Span ID: span-002
154
+ Type: custom
155
+ Status: success
156
+ Attributes:
157
+ stream.chunks: 12
158
+ ```
159
+
160
+ ---
161
+
162
+ ### `prela trace search`
163
+
164
+ Search span names and attributes for matching traces.
165
+
166
+ **Usage:**
167
+ ```bash
168
+ prela trace search QUERY
169
+ ```
170
+
171
+ **Arguments:**
172
+ - `QUERY` - Search query (case-insensitive)
173
+
174
+ **Search Scope:**
175
+ - Span names
176
+ - Attribute keys
177
+ - Attribute values
178
+
179
+ **Output:**
180
+ Table showing matching traces with:
181
+ - Trace ID
182
+ - Root span name
183
+ - Number of matching spans
184
+ - Status
185
+
186
+ **Example:**
187
+ ```bash
188
+ $ prela trace search anthropic
189
+
190
+ Found 3 traces matching 'anthropic'
191
+
192
+ ┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━┳━━━━━━━━┓
193
+ ┃ Trace ID ┃ Root Span ┃ Matching Spans ┃ Status ┃
194
+ ┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━╇━━━━━━━━┩
195
+ │ trace-abc123... │ anthropic.messages │ 2 │ ✓ │
196
+ │ trace-def456... │ langchain.agent │ 1 │ ✓ │
197
+ │ trace-ghi789... │ anthropic.stream │ 3 │ ✓ │
198
+ └──────────────────┴────────────────────────┴─────────────────┴────────┘
199
+ ```
200
+
201
+ ---
202
+
203
+ ### `prela serve`
204
+
205
+ Start local web dashboard (placeholder - not yet implemented).
206
+
207
+ **Usage:**
208
+ ```bash
209
+ prela serve [OPTIONS]
210
+ ```
211
+
212
+ **Options:**
213
+ - `--port, -p INTEGER` - Port to run server on (default: 8000)
214
+
215
+ **Status:**
216
+ This feature is planned for Phase 1 (Months 4-8). When implemented, it will:
217
+ - Start FastAPI server on specified port
218
+ - Serve REST API endpoints for trace retrieval
219
+ - Serve static React frontend for trace visualization
220
+ - Provide real-time trace streaming (WebSocket)
221
+
222
+ **Example:**
223
+ ```bash
224
+ $ prela serve --port 8000
225
+
226
+ Web dashboard not yet implemented
227
+ Planned: Start server on http://localhost:8000
228
+ Will serve API endpoints + static frontend
229
+
230
+ This feature is planned for Phase 1 (Months 4-8)
231
+ ```
232
+
233
+ ---
234
+
235
+ ### `prela eval`
236
+
237
+ Run evaluation suite (placeholder - not yet implemented).
238
+
239
+ **Usage:**
240
+ ```bash
241
+ prela eval SUITE_PATH
242
+ ```
243
+
244
+ **Arguments:**
245
+ - `SUITE_PATH` - Path to evaluation suite file (YAML or JSON)
246
+
247
+ **Status:**
248
+ This feature is planned for Phase 1 (Months 4-8). When implemented, it will:
249
+ - Load evaluation suite from file
250
+ - Run test cases against traced LLM calls
251
+ - Compare outputs against expected results
252
+ - Generate pass/fail metrics and reports
253
+
254
+ **Planned Suite Format:**
255
+ ```yaml
256
+ name: my-eval-suite
257
+ test_cases:
258
+ - name: factual_accuracy
259
+ input: "What is the capital of France?"
260
+ expected_output: "Paris"
261
+ criteria:
262
+ - type: exact_match
263
+ - type: semantic_similarity
264
+ threshold: 0.9
265
+
266
+ - name: hallucination_check
267
+ input: "Explain quantum computing"
268
+ criteria:
269
+ - type: no_hallucination
270
+ - type: coherence
271
+ ```
272
+
273
+ **Example:**
274
+ ```bash
275
+ $ prela eval suite.yaml
276
+
277
+ Eval runner not yet implemented
278
+ Planned: Run eval suite from suite.yaml
279
+ Will output results with pass/fail metrics
280
+
281
+ This feature is planned for Phase 1 (Months 4-8)
282
+ ```
283
+
284
+ ---
285
+
286
+ ## Configuration File
287
+
288
+ The `.prela.yaml` configuration file is created by `prela init` and used by all commands.
289
+
290
+ **Location:**
291
+ Current working directory (`.prela.yaml`)
292
+
293
+ **Format:**
294
+ ```yaml
295
+ service_name: my-agent # Service identifier
296
+ exporter: file # Exporter type (console/file)
297
+ trace_dir: ./traces # Trace directory (file exporter only)
298
+ sample_rate: 1.0 # Sampling rate (0.0-1.0)
299
+ ```
300
+
301
+ **Default Values:**
302
+ - `service_name`: `my-agent`
303
+ - `exporter`: `file`
304
+ - `trace_dir`: `./traces`
305
+ - `sample_rate`: `1.0` (100%)
306
+
307
+ **Manual Editing:**
308
+ You can manually edit `.prela.yaml` instead of using `prela init`.
309
+
310
+ ---
311
+
312
+ ## Examples
313
+
314
+ ### Basic Workflow
315
+
316
+ ```bash
317
+ # 1. Initialize configuration
318
+ $ prela init
319
+ Service name [my-agent]: my-chatbot
320
+ Exporter (console/file) [file]: file
321
+ Trace directory [./traces]: ./traces
322
+ Sample rate (0.0-1.0) [1.0]: 1.0
323
+
324
+ # 2. Run your application (generates traces)
325
+ $ python my_chatbot.py
326
+
327
+ # 3. List recent traces
328
+ $ prela trace list --limit 5
329
+
330
+ # 4. Show detailed trace
331
+ $ prela trace show trace-abc123
332
+
333
+ # 5. Search for specific traces
334
+ $ prela trace search error
335
+ ```
336
+
337
+ ### Development Workflow
338
+
339
+ ```bash
340
+ # Use console exporter for development
341
+ $ prela init
342
+ Exporter (console/file) [file]: console
343
+
344
+ # Run your app with live trace output
345
+ $ python my_app.py
346
+ ```
347
+
348
+ ### Production Monitoring
349
+
350
+ ```bash
351
+ # Sample 10% of traces in production
352
+ $ prela init
353
+ Sample rate (0.0-1.0) [1.0]: 0.1
354
+
355
+ # List traces from last hour
356
+ $ prela trace list --since 1h
357
+
358
+ # Search for errors
359
+ $ prela trace search error
360
+ ```
361
+
362
+ ---
363
+
364
+ ## Troubleshooting
365
+
366
+ ### "CLI dependencies not installed"
367
+
368
+ Install CLI dependencies:
369
+ ```bash
370
+ pip install prela[cli]
371
+ ```
372
+
373
+ Or install individually:
374
+ ```bash
375
+ pip install typer pyyaml rich
376
+ ```
377
+
378
+ ### "No traces found"
379
+
380
+ Check:
381
+ 1. Trace directory exists and contains `.jsonl` files
382
+ 2. `.prela.yaml` points to correct `trace_dir`
383
+ 3. Application is actually generating traces
384
+
385
+ ### "Configuration saved" but file not found
386
+
387
+ Check that you're in the correct directory. `.prela.yaml` is created in the current working directory.
388
+
389
+ ---
390
+
391
+ ## Architecture
392
+
393
+ The CLI is built with:
394
+ - **Typer** - Modern CLI framework with type hints
395
+ - **Rich** - Beautiful terminal output (tables, trees, colors)
396
+ - **PyYAML** - YAML configuration parsing
397
+
398
+ **File Structure:**
399
+ ```
400
+ prela/contrib/
401
+ ├── __init__.py
402
+ ├── cli.py # CLI implementation
403
+ └── CLI.md # This documentation
404
+
405
+ tests/
406
+ └── test_cli.py # CLI tests (28 tests)
407
+ ```
408
+
409
+ **Entry Point:**
410
+ Registered in `pyproject.toml`:
411
+ ```toml
412
+ [project.scripts]
413
+ prela = "prela.contrib.cli:main"
414
+ ```
415
+
416
+ ---
417
+
418
+ ## Future Enhancements
419
+
420
+ Planned for Phase 1 (Months 4-8):
421
+ - `prela serve` - Local web dashboard
422
+ - `prela eval` - Evaluation suite runner
423
+ - `prela export` - Export traces to different formats
424
+ - `prela stats` - Aggregate statistics and metrics
425
+ - `prela compare` - Compare traces side-by-side
426
+
427
+ Planned for Phase 2:
428
+ - `prela cloud push` - Push traces to cloud platform
429
+ - `prela cloud sync` - Sync local and cloud traces
430
+ - `prela alert` - Configure alerting rules
431
+ - `prela team` - Team collaboration features
@@ -0,0 +1,118 @@
1
+ # Prela CLI
2
+
3
+ Command-line interface for the Prela AI Agent Observability Platform.
4
+
5
+ ## Quick Start
6
+
7
+ ### Installation
8
+
9
+ ```bash
10
+ pip install prela[cli]
11
+ ```
12
+
13
+ ### Basic Usage
14
+
15
+ ```bash
16
+ # 1. Initialize configuration
17
+ prela init
18
+
19
+ # 2. Run your AI application (generates traces)
20
+ python my_agent.py
21
+
22
+ # 3. View traces
23
+ prela trace list
24
+ prela trace show <trace-id>
25
+ prela trace search <query>
26
+ ```
27
+
28
+ ## Commands
29
+
30
+ | Command | Description |
31
+ |---------|-------------|
32
+ | `prela init` | Initialize configuration with interactive prompts |
33
+ | `prela trace list` | List recent traces |
34
+ | `prela trace show` | Display full trace tree |
35
+ | `prela trace search` | Search traces by name or attributes |
36
+ | `prela serve` | Start local web dashboard (coming soon) |
37
+ | `prela eval` | Run evaluation suite (coming soon) |
38
+
39
+ ## Documentation
40
+
41
+ See [CLI.md](CLI.md) for complete documentation.
42
+
43
+ ## Example
44
+
45
+ ```bash
46
+ $ prela init
47
+ Service name [my-agent]: my-chatbot
48
+ Exporter (console/file) [file]: file
49
+ Trace directory [./traces]: ./traces
50
+ Sample rate (0.0-1.0) [1.0]: 1.0
51
+
52
+ ✓ Configuration saved to .prela.yaml
53
+ ✓ Created trace directory: ./traces
54
+
55
+ $ python examples/cli_demo.py
56
+ Prela initialized with file exporter
57
+ Generating sample traces...
58
+ ✓ Created trace: example_operation
59
+ ✓ Created trace: parent_task (with 2 child spans)
60
+
61
+ $ prela trace list --limit 5
62
+
63
+ ┏━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━┳━━━━━━━━┳━━━━━━━┳━━━━━━━━━━━━━━━━━━━━━┓
64
+ ┃ Trace ID ┃ Root Span ┃ Duration ┃ Status ┃ Spans ┃ Time ┃
65
+ ┡━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━╇━━━━━━━━╇━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━┩
66
+ │ trace-abc123... │ example_operation │ 123ms │ ✓ │ 1 │ 2025-01-26 10:30:45 │
67
+ │ trace-def456... │ parent_task │ 234ms │ ✓ │ 3 │ 2025-01-26 10:30:46 │
68
+ └──────────────────┴────────────────────┴──────────┴────────┴───────┴─────────────────────┘
69
+
70
+ $ prela trace show trace-abc123
71
+
72
+ Trace: trace-abc123456789
73
+
74
+ example_operation (custom) success 123ms
75
+
76
+ Span Details:
77
+
78
+ example_operation
79
+ Span ID: span-001
80
+ Type: custom
81
+ Status: success
82
+ Attributes:
83
+ operation.type: demo
84
+ operation.complexity: simple
85
+ Events (2):
86
+ - operation.started @ 2025-01-26T10:30:45.123456Z
87
+ - operation.completed @ 2025-01-26T10:30:45.246789Z
88
+ ```
89
+
90
+ ## Features
91
+
92
+ - ✅ **Configuration Management** - Interactive setup with `.prela.yaml`
93
+ - ✅ **Trace Listing** - View recent traces with filtering and pagination
94
+ - ✅ **Trace Details** - Full trace tree visualization with Rich
95
+ - ✅ **Search** - Find traces by span names or attributes
96
+ - ✅ **Time Filtering** - View traces from specific time ranges
97
+ - ⏳ **Web Dashboard** - Coming in Phase 1
98
+ - ⏳ **Eval Suite** - Coming in Phase 1
99
+
100
+ ## Requirements
101
+
102
+ - Python 3.9+
103
+ - typer >= 0.9.0
104
+ - pyyaml >= 6.0
105
+ - rich >= 13.0.0
106
+
107
+ ## Development
108
+
109
+ Run tests:
110
+ ```bash
111
+ pytest tests/test_cli.py -v
112
+ ```
113
+
114
+ All 39 tests passing ✅
115
+
116
+ ## License
117
+
118
+ Apache 2.0
@@ -0,0 +1,5 @@
1
+ """Prela contrib modules - optional tools and utilities."""
2
+
3
+ from __future__ import annotations
4
+
5
+ __all__ = []