toad-eye 0.1.0 → 0.2.0
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +129 -0
- package/package.json +12 -3
package/README.md
ADDED
|
@@ -0,0 +1,129 @@
|
|
|
1
|
+
# toad-eye 🐸👁️
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
OpenTelemetry-based observability toolkit for LLM systems.
|
|
14
|
+
|
|
15
|
+
One-line instrumentation for any LLM service — get traces, metrics, and Grafana dashboards out of the box.
|
|
16
|
+
|
|
17
|
+
## Why toad-eye?
|
|
18
|
+
|
|
19
|
+
LLM APIs are black boxes — you don't know what they cost, how slow they are, or why they fail. toad-eye gives you full visibility with one line of code.
|
|
20
|
+
|
|
21
|
+
## Architecture
|
|
22
|
+
|
|
23
|
+
```mermaid
|
|
24
|
+
flowchart LR
|
|
25
|
+
App["🔮 Your LLM service"]
|
|
26
|
+
Inst["🐸 @toad-eye/instrumentation"]
|
|
27
|
+
Coll["📡 OTel Collector"]
|
|
28
|
+
Prom["📊 Prometheus"]
|
|
29
|
+
Jaeger["🔍 Jaeger"]
|
|
30
|
+
Graf["📈 Grafana"]
|
|
31
|
+
|
|
32
|
+
App --> Inst --> Coll
|
|
33
|
+
Coll --> Prom --> Graf
|
|
34
|
+
Coll --> Jaeger
|
|
35
|
+
|
|
36
|
+
style App fill:#4a5568,stroke:#718096,color:#fff
|
|
37
|
+
style Inst fill:#2d6a4f,stroke:#40916c,color:#fff
|
|
38
|
+
style Coll fill:#1d4ed8,stroke:#3b82f6,color:#fff
|
|
39
|
+
style Prom fill:#e24d1e,stroke:#ff6633,color:#fff
|
|
40
|
+
style Jaeger fill:#00bcd4,stroke:#26c6da,color:#fff
|
|
41
|
+
style Graf fill:#f46800,stroke:#ff8c00,color:#fff
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
## Quick start
|
|
45
|
+
|
|
46
|
+
```bash
|
|
47
|
+
npm install
|
|
48
|
+
cd infra && docker compose up -d # start observability stack
|
|
49
|
+
npm run demo # start mock LLM service
|
|
50
|
+
npm run load --workspace=demo # send test traffic
|
|
51
|
+
```
|
|
52
|
+
|
|
53
|
+
| Service | URL |
|
|
54
|
+
| -------------- | ----------------------------------------- |
|
|
55
|
+
| Grafana | [localhost:3000](http://localhost:3000) |
|
|
56
|
+
| Jaeger UI | [localhost:16686](http://localhost:16686) |
|
|
57
|
+
| Prometheus | [localhost:9090](http://localhost:9090) |
|
|
58
|
+
| OTel Collector | [localhost:4318](http://localhost:4318) |
|
|
59
|
+
|
|
60
|
+
## Usage
|
|
61
|
+
|
|
62
|
+
```typescript
|
|
63
|
+
import { initObservability, traceLLMCall } from "@toad-eye/instrumentation";
|
|
64
|
+
|
|
65
|
+
// one-line init
|
|
66
|
+
initObservability({
|
|
67
|
+
serviceName: "my-llm-service",
|
|
68
|
+
endpoint: "http://localhost:4318",
|
|
69
|
+
});
|
|
70
|
+
|
|
71
|
+
// wrap any LLM call
|
|
72
|
+
const result = await traceLLMCall(
|
|
73
|
+
{ provider: "anthropic", model: "claude-sonnet-4-20250514", prompt: "hello" },
|
|
74
|
+
() => callYourLLM(),
|
|
75
|
+
);
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
> **Privacy mode:** pass `recordContent: false` to `initObservability()` to stop recording prompt/completion text in spans. Useful in production when prompts contain sensitive data.
|
|
79
|
+
|
|
80
|
+
## What it tracks
|
|
81
|
+
|
|
82
|
+
### Metrics
|
|
83
|
+
|
|
84
|
+
| Metric | Type | Description |
|
|
85
|
+
| ---------------------- | --------- | -------------------------------------- |
|
|
86
|
+
| `llm.request.duration` | Histogram | Request latency in milliseconds |
|
|
87
|
+
| `llm.request.cost` | Histogram | Cost per request in USD |
|
|
88
|
+
| `llm.tokens` | Counter | Total tokens consumed (input + output) |
|
|
89
|
+
| `llm.requests` | Counter | Total requests made |
|
|
90
|
+
| `llm.errors` | Counter | Total failed requests |
|
|
91
|
+
|
|
92
|
+
All metrics are labeled with `provider` and `model` for filtering and grouping.
|
|
93
|
+
|
|
94
|
+
### Span attributes
|
|
95
|
+
|
|
96
|
+
| Attribute | Type | Description |
|
|
97
|
+
| ------------------- | ------ | ------------------------------- |
|
|
98
|
+
| `llm.provider` | string | `anthropic`, `gemini`, `openai` |
|
|
99
|
+
| `llm.model` | string | Model identifier |
|
|
100
|
+
| `llm.prompt` | string | Prompt sent to the LLM |
|
|
101
|
+
| `llm.completion` | string | Response from the LLM |
|
|
102
|
+
| `llm.input_tokens` | number | Tokens in the prompt |
|
|
103
|
+
| `llm.output_tokens` | number | Tokens in the completion |
|
|
104
|
+
| `llm.cost` | number | Cost in USD |
|
|
105
|
+
| `llm.temperature` | number | Temperature parameter |
|
|
106
|
+
| `llm.status` | string | `success` or `error` |
|
|
107
|
+
| `llm.error` | string | Error message (if failed) |
|
|
108
|
+
|
|
109
|
+
## Grafana dashboard
|
|
110
|
+
|
|
111
|
+
|
|
112
|
+
|
|
113
|
+
## Jaeger traces
|
|
114
|
+
|
|
115
|
+
|
|
116
|
+
|
|
117
|
+
## Project structure
|
|
118
|
+
|
|
119
|
+
```
|
|
120
|
+
packages/instrumentation/ — NPM package (@toad-eye/instrumentation)
|
|
121
|
+
demo/ — mock LLM service for testing
|
|
122
|
+
infra/ — docker-compose stack (OTel + Prometheus + Jaeger + Grafana)
|
|
123
|
+
```
|
|
124
|
+
|
|
125
|
+
## Tech stack
|
|
126
|
+
|
|
127
|
+
- TypeScript, OpenTelemetry SDK 2.x, OTLP exporters
|
|
128
|
+
- Hono (demo server)
|
|
129
|
+
- Docker Compose (Prometheus, Jaeger, Grafana, OTel Collector)
|
package/package.json
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "toad-eye",
|
|
3
|
-
"version": "0.
|
|
3
|
+
"version": "0.2.0",
|
|
4
4
|
"description": "One-line OpenTelemetry instrumentation for LLM services",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
7
7
|
"types": "src/index.ts",
|
|
8
|
-
"
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/albertalov/toad-eye"
|
|
11
|
+
},
|
|
12
|
+
"homepage": "https://github.com/albertalov/toad-eye#readme",
|
|
13
|
+
"author": "albertalov",
|
|
9
14
|
"keywords": [
|
|
10
15
|
"opentelemetry",
|
|
11
16
|
"llm",
|
|
12
|
-
"instrumentation"
|
|
17
|
+
"instrumentation",
|
|
18
|
+
"observability",
|
|
19
|
+
"tracing",
|
|
20
|
+
"metrics",
|
|
21
|
+
"grafana"
|
|
13
22
|
],
|
|
14
23
|
"license": "ISC",
|
|
15
24
|
"dependencies": {
|