toad-eye 0.1.0 β 0.2.1
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- package/README.md +120 -0
- package/package.json +12 -3
package/README.md
ADDED
|
@@ -0,0 +1,120 @@
|
|
|
1
|
+
# toad-eye πΈποΈ
|
|
2
|
+
|
|
3
|
+
|
|
4
|
+
|
|
5
|
+
|
|
6
|
+
|
|
7
|
+
|
|
8
|
+
|
|
9
|
+
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
|
|
13
|
+
OpenTelemetry-based observability toolkit for LLM systems.
|
|
14
|
+
|
|
15
|
+
One-line instrumentation for any LLM service β get traces, metrics, and Grafana dashboards out of the box.
|
|
16
|
+
|
|
17
|
+
## Why toad-eye?
|
|
18
|
+
|
|
19
|
+
LLM APIs are black boxes β you don't know what they cost, how slow they are, or why they fail. toad-eye gives you full visibility with one line of code.
|
|
20
|
+
|
|
21
|
+
## Architecture
|
|
22
|
+
|
|
23
|
+
```
|
|
24
|
+
Your LLM service
|
|
25
|
+
β
|
|
26
|
+
βΌ
|
|
27
|
+
@toad-eye/instrumentation
|
|
28
|
+
β
|
|
29
|
+
βΌ OTLP/HTTP
|
|
30
|
+
OTel Collector
|
|
31
|
+
ββββΆ Prometheus βββΆ Grafana
|
|
32
|
+
ββββΆ Jaeger
|
|
33
|
+
```
|
|
34
|
+
|
|
35
|
+
## Quick start
|
|
36
|
+
|
|
37
|
+
```bash
|
|
38
|
+
npm install
|
|
39
|
+
cd infra && docker compose up -d # start observability stack
|
|
40
|
+
npm run demo # start mock LLM service
|
|
41
|
+
npm run load --workspace=demo # send test traffic
|
|
42
|
+
```
|
|
43
|
+
|
|
44
|
+
| Service | URL |
|
|
45
|
+
| -------------- | ----------------------------------------- |
|
|
46
|
+
| Grafana | [localhost:3000](http://localhost:3000) |
|
|
47
|
+
| Jaeger UI | [localhost:16686](http://localhost:16686) |
|
|
48
|
+
| Prometheus | [localhost:9090](http://localhost:9090) |
|
|
49
|
+
| OTel Collector | [localhost:4318](http://localhost:4318) |
|
|
50
|
+
|
|
51
|
+
## Usage
|
|
52
|
+
|
|
53
|
+
```typescript
|
|
54
|
+
import { initObservability, traceLLMCall } from "@toad-eye/instrumentation";
|
|
55
|
+
|
|
56
|
+
// one-line init
|
|
57
|
+
initObservability({
|
|
58
|
+
serviceName: "my-llm-service",
|
|
59
|
+
endpoint: "http://localhost:4318",
|
|
60
|
+
});
|
|
61
|
+
|
|
62
|
+
// wrap any LLM call
|
|
63
|
+
const result = await traceLLMCall(
|
|
64
|
+
{ provider: "anthropic", model: "claude-sonnet-4-20250514", prompt: "hello" },
|
|
65
|
+
() => callYourLLM(),
|
|
66
|
+
);
|
|
67
|
+
```
|
|
68
|
+
|
|
69
|
+
> **Privacy mode:** pass `recordContent: false` to `initObservability()` to stop recording prompt/completion text in spans. Useful in production when prompts contain sensitive data.
|
|
70
|
+
|
|
71
|
+
## What it tracks
|
|
72
|
+
|
|
73
|
+
### Metrics
|
|
74
|
+
|
|
75
|
+
| Metric | Type | Description |
|
|
76
|
+
| ---------------------- | --------- | -------------------------------------- |
|
|
77
|
+
| `llm.request.duration` | Histogram | Request latency in milliseconds |
|
|
78
|
+
| `llm.request.cost` | Histogram | Cost per request in USD |
|
|
79
|
+
| `llm.tokens` | Counter | Total tokens consumed (input + output) |
|
|
80
|
+
| `llm.requests` | Counter | Total requests made |
|
|
81
|
+
| `llm.errors` | Counter | Total failed requests |
|
|
82
|
+
|
|
83
|
+
All metrics are labeled with `provider` and `model` for filtering and grouping.
|
|
84
|
+
|
|
85
|
+
### Span attributes
|
|
86
|
+
|
|
87
|
+
| Attribute | Type | Description |
|
|
88
|
+
| ------------------- | ------ | ------------------------------- |
|
|
89
|
+
| `llm.provider` | string | `anthropic`, `gemini`, `openai` |
|
|
90
|
+
| `llm.model` | string | Model identifier |
|
|
91
|
+
| `llm.prompt` | string | Prompt sent to the LLM |
|
|
92
|
+
| `llm.completion` | string | Response from the LLM |
|
|
93
|
+
| `llm.input_tokens` | number | Tokens in the prompt |
|
|
94
|
+
| `llm.output_tokens` | number | Tokens in the completion |
|
|
95
|
+
| `llm.cost` | number | Cost in USD |
|
|
96
|
+
| `llm.temperature` | number | Temperature parameter |
|
|
97
|
+
| `llm.status` | string | `success` or `error` |
|
|
98
|
+
| `llm.error` | string | Error message (if failed) |
|
|
99
|
+
|
|
100
|
+
## Grafana dashboard
|
|
101
|
+
|
|
102
|
+
|
|
103
|
+
|
|
104
|
+
## Jaeger traces
|
|
105
|
+
|
|
106
|
+
|
|
107
|
+
|
|
108
|
+
## Project structure
|
|
109
|
+
|
|
110
|
+
```
|
|
111
|
+
packages/instrumentation/ β NPM package (@toad-eye/instrumentation)
|
|
112
|
+
demo/ β mock LLM service for testing
|
|
113
|
+
infra/ β docker-compose stack (OTel + Prometheus + Jaeger + Grafana)
|
|
114
|
+
```
|
|
115
|
+
|
|
116
|
+
## Tech stack
|
|
117
|
+
|
|
118
|
+
- TypeScript, OpenTelemetry SDK 2.x, OTLP exporters
|
|
119
|
+
- Hono (demo server)
|
|
120
|
+
- Docker Compose (Prometheus, Jaeger, Grafana, OTel Collector)
|
package/package.json
CHANGED
|
@@ -1,15 +1,24 @@
|
|
|
1
1
|
{
|
|
2
2
|
"name": "toad-eye",
|
|
3
|
-
"version": "0.1
|
|
3
|
+
"version": "0.2.1",
|
|
4
4
|
"description": "One-line OpenTelemetry instrumentation for LLM services",
|
|
5
5
|
"type": "module",
|
|
6
6
|
"main": "src/index.ts",
|
|
7
7
|
"types": "src/index.ts",
|
|
8
|
-
"
|
|
8
|
+
"repository": {
|
|
9
|
+
"type": "git",
|
|
10
|
+
"url": "https://github.com/albertalov/toad-eye"
|
|
11
|
+
},
|
|
12
|
+
"homepage": "https://github.com/albertalov/toad-eye#readme",
|
|
13
|
+
"author": "albertalov",
|
|
9
14
|
"keywords": [
|
|
10
15
|
"opentelemetry",
|
|
11
16
|
"llm",
|
|
12
|
-
"instrumentation"
|
|
17
|
+
"instrumentation",
|
|
18
|
+
"observability",
|
|
19
|
+
"tracing",
|
|
20
|
+
"metrics",
|
|
21
|
+
"grafana"
|
|
13
22
|
],
|
|
14
23
|
"license": "ISC",
|
|
15
24
|
"dependencies": {
|