tokenjam 0.2.0__py3-none-any.whl
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- tokenjam/__init__.py +1 -0
- tokenjam/api/__init__.py +0 -0
- tokenjam/api/app.py +104 -0
- tokenjam/api/deps.py +18 -0
- tokenjam/api/middleware.py +28 -0
- tokenjam/api/routes/__init__.py +0 -0
- tokenjam/api/routes/agents.py +33 -0
- tokenjam/api/routes/alerts.py +77 -0
- tokenjam/api/routes/budget.py +96 -0
- tokenjam/api/routes/cost.py +43 -0
- tokenjam/api/routes/drift.py +63 -0
- tokenjam/api/routes/logs.py +511 -0
- tokenjam/api/routes/metrics.py +81 -0
- tokenjam/api/routes/otlp.py +63 -0
- tokenjam/api/routes/spans.py +202 -0
- tokenjam/api/routes/status.py +84 -0
- tokenjam/api/routes/tools.py +22 -0
- tokenjam/api/routes/traces.py +92 -0
- tokenjam/cli/__init__.py +0 -0
- tokenjam/cli/cmd_alerts.py +94 -0
- tokenjam/cli/cmd_budget.py +119 -0
- tokenjam/cli/cmd_cost.py +90 -0
- tokenjam/cli/cmd_demo.py +82 -0
- tokenjam/cli/cmd_doctor.py +173 -0
- tokenjam/cli/cmd_drift.py +238 -0
- tokenjam/cli/cmd_export.py +200 -0
- tokenjam/cli/cmd_mcp.py +78 -0
- tokenjam/cli/cmd_onboard.py +779 -0
- tokenjam/cli/cmd_serve.py +85 -0
- tokenjam/cli/cmd_status.py +153 -0
- tokenjam/cli/cmd_stop.py +87 -0
- tokenjam/cli/cmd_tools.py +45 -0
- tokenjam/cli/cmd_traces.py +161 -0
- tokenjam/cli/cmd_uninstall.py +159 -0
- tokenjam/cli/main.py +110 -0
- tokenjam/core/__init__.py +0 -0
- tokenjam/core/alerts.py +619 -0
- tokenjam/core/api_backend.py +235 -0
- tokenjam/core/config.py +360 -0
- tokenjam/core/cost.py +102 -0
- tokenjam/core/db.py +718 -0
- tokenjam/core/drift.py +256 -0
- tokenjam/core/ingest.py +265 -0
- tokenjam/core/models.py +225 -0
- tokenjam/core/pricing.py +54 -0
- tokenjam/core/retention.py +21 -0
- tokenjam/core/schema_validator.py +156 -0
- tokenjam/demo/__init__.py +0 -0
- tokenjam/demo/env.py +96 -0
- tokenjam/mcp/__init__.py +0 -0
- tokenjam/mcp/server.py +1067 -0
- tokenjam/otel/__init__.py +0 -0
- tokenjam/otel/exporters.py +26 -0
- tokenjam/otel/provider.py +207 -0
- tokenjam/otel/semconv.py +144 -0
- tokenjam/pricing/models.toml +70 -0
- tokenjam/py.typed +0 -0
- tokenjam/sdk/__init__.py +21 -0
- tokenjam/sdk/agent.py +206 -0
- tokenjam/sdk/bootstrap.py +120 -0
- tokenjam/sdk/http_exporter.py +109 -0
- tokenjam/sdk/integrations/__init__.py +0 -0
- tokenjam/sdk/integrations/anthropic.py +200 -0
- tokenjam/sdk/integrations/autogen.py +97 -0
- tokenjam/sdk/integrations/base.py +27 -0
- tokenjam/sdk/integrations/bedrock.py +103 -0
- tokenjam/sdk/integrations/crewai.py +96 -0
- tokenjam/sdk/integrations/gemini.py +131 -0
- tokenjam/sdk/integrations/langchain.py +156 -0
- tokenjam/sdk/integrations/langgraph.py +101 -0
- tokenjam/sdk/integrations/litellm.py +323 -0
- tokenjam/sdk/integrations/llamaindex.py +52 -0
- tokenjam/sdk/integrations/nemoclaw.py +139 -0
- tokenjam/sdk/integrations/openai.py +159 -0
- tokenjam/sdk/integrations/openai_agents_sdk.py +47 -0
- tokenjam/sdk/transport.py +98 -0
- tokenjam/ui/index.html +1213 -0
- tokenjam/utils/__init__.py +0 -0
- tokenjam/utils/formatting.py +43 -0
- tokenjam/utils/ids.py +15 -0
- tokenjam/utils/time_parse.py +54 -0
- tokenjam-0.2.0.dist-info/METADATA +622 -0
- tokenjam-0.2.0.dist-info/RECORD +86 -0
- tokenjam-0.2.0.dist-info/WHEEL +4 -0
- tokenjam-0.2.0.dist-info/entry_points.txt +2 -0
- tokenjam-0.2.0.dist-info/licenses/LICENSE +21 -0
|
@@ -0,0 +1,622 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: tokenjam
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: TokenJam — local-first OTel-native observability for Autonomous AI agents
|
|
5
|
+
Project-URL: Homepage, https://opencla.watch
|
|
6
|
+
Project-URL: Repository, https://github.com/Metabuilder-Labs/openclawwatch
|
|
7
|
+
Project-URL: Issues, https://github.com/Metabuilder-Labs/openclawwatch/issues
|
|
8
|
+
Author-email: Anil Murty <anil@metabldr.com>
|
|
9
|
+
License: MIT
|
|
10
|
+
License-File: LICENSE
|
|
11
|
+
Keywords: agents,ai,llm,observability,opentelemetry
|
|
12
|
+
Classifier: Development Status :: 4 - Beta
|
|
13
|
+
Classifier: Intended Audience :: Developers
|
|
14
|
+
Classifier: License :: OSI Approved :: MIT License
|
|
15
|
+
Classifier: Programming Language :: Python :: 3
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries
|
|
20
|
+
Classifier: Topic :: System :: Monitoring
|
|
21
|
+
Requires-Python: >=3.10
|
|
22
|
+
Requires-Dist: apscheduler>=3.10
|
|
23
|
+
Requires-Dist: click>=8.1
|
|
24
|
+
Requires-Dist: duckdb>=0.10
|
|
25
|
+
Requires-Dist: fastapi>=0.110
|
|
26
|
+
Requires-Dist: genson>=1.2
|
|
27
|
+
Requires-Dist: httpx>=0.27
|
|
28
|
+
Requires-Dist: jsonschema>=4.0
|
|
29
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.25
|
|
30
|
+
Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.25
|
|
31
|
+
Requires-Dist: opentelemetry-exporter-prometheus>=0.46b0
|
|
32
|
+
Requires-Dist: opentelemetry-sdk>=1.25
|
|
33
|
+
Requires-Dist: pytz>=2024.1
|
|
34
|
+
Requires-Dist: rich>=13.0
|
|
35
|
+
Requires-Dist: tomli-w>=1.0
|
|
36
|
+
Requires-Dist: tomli>=2.0; python_version < '3.11'
|
|
37
|
+
Requires-Dist: uvicorn>=0.27
|
|
38
|
+
Requires-Dist: websockets>=12.0
|
|
39
|
+
Provides-Extra: autogen
|
|
40
|
+
Requires-Dist: pyautogen>=0.2; extra == 'autogen'
|
|
41
|
+
Provides-Extra: crewai
|
|
42
|
+
Requires-Dist: crewai>=0.28; extra == 'crewai'
|
|
43
|
+
Provides-Extra: dev
|
|
44
|
+
Requires-Dist: httpx; extra == 'dev'
|
|
45
|
+
Requires-Dist: mypy; extra == 'dev'
|
|
46
|
+
Requires-Dist: pytest; extra == 'dev'
|
|
47
|
+
Requires-Dist: pytest-asyncio; extra == 'dev'
|
|
48
|
+
Requires-Dist: ruff; extra == 'dev'
|
|
49
|
+
Provides-Extra: langchain
|
|
50
|
+
Requires-Dist: langchain>=0.2; extra == 'langchain'
|
|
51
|
+
Provides-Extra: litellm
|
|
52
|
+
Requires-Dist: litellm>=1.40; extra == 'litellm'
|
|
53
|
+
Provides-Extra: mcp
|
|
54
|
+
Requires-Dist: fastmcp; extra == 'mcp'
|
|
55
|
+
Description-Content-Type: text/markdown
|
|
56
|
+
|
|
57
|
+
<div align="center">
|
|
58
|
+
|
|
59
|
+
<img src="https://opencla.watch/icon.svg" alt="TokenJam" width="72" height="72">
|
|
60
|
+
|
|
61
|
+
# TokenJam
|
|
62
|
+
|
|
63
|
+
The open-source LLM observability tool for autonomous agents.
|
|
64
|
+
|
|
65
|
+
No cloud. No signup. No surprises.
|
|
66
|
+
|
|
67
|
+
[](https://github.com/Metabuilder-Labs/tokenjam/actions/workflows/ci.yml)
|
|
68
|
+
[](https://pypi.org/project/tokenjam/)
|
|
69
|
+
[](https://pypi.org/project/tokenjam/)
|
|
70
|
+
[](https://www.npmjs.com/package/@tokenjam/sdk)
|
|
71
|
+
[](LICENSE)
|
|
72
|
+
[](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
|
|
73
|
+
|
|
74
|
+
```
|
|
75
|
+
pip install tokenjam
|
|
76
|
+
```
|
|
77
|
+
|
|
78
|
+
</div>
|
|
79
|
+
|
|
80
|
+
---
|
|
81
|
+
|
|
82
|
+
Your agent sends emails, writes files, calls APIs, and spends your money — all while you're away. Most observability tools were built for LLM developers building chat products. `tj` was built for **agents with real-world consequences**: real-time cost tracking, safety alerts, behavioral drift detection, all running locally on your machine.
|
|
83
|
+
|
|
84
|
+
---
|
|
85
|
+
|
|
86
|
+
## What you get
|
|
87
|
+
|
|
88
|
+
**Real-time cost tracking.** Every LLM call is priced as it happens — by agent, model, session, and tool. Budget alerts fire before you hit the limit, not after.
|
|
89
|
+
|
|
90
|
+
**Safety alerts.** Configure any tool call as a sensitive action (`send_email`, `delete_file`, `submit_form`) and get notified instantly via ntfy, Discord, Telegram, webhook, or stdout.
|
|
91
|
+
|
|
92
|
+
**Behavioral drift detection.** `tj` builds a statistical baseline from your agent's real behavior and alerts when something deviates — a prompt tweak, a model update, a dependency bump. No LLM required.
|
|
93
|
+
|
|
94
|
+
**Tool output validation.** Declare a JSON Schema for your tools or let `tj` infer one automatically. Schema violations are caught the moment they occur.
|
|
95
|
+
|
|
96
|
+
**100% local.** DuckDB. Local REST API. No cloud backend. No API key for `tj` itself. Your telemetry never leaves your machine unless you explicitly export it.
|
|
97
|
+
|
|
98
|
+
---
|
|
99
|
+
|
|
100
|
+
## Get started
|
|
101
|
+
|
|
102
|
+
`tj` works four ways. Pick the one that fits.
|
|
103
|
+
|
|
104
|
+
### Coding agents — zero code
|
|
105
|
+
|
|
106
|
+
For **Claude Code**, **Codex**, and any agent that already emits OpenTelemetry. No SDK, no code changes.
|
|
107
|
+
|
|
108
|
+
```bash
|
|
109
|
+
pip install "tokenjam[mcp]"
|
|
110
|
+
tj onboard --claude-code # or: tj onboard --codex
|
|
111
|
+
# Restart your coding agent
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
Every session, API call, tool use, and error is now a tracked span with cost and alert evaluation. The MCP server gives your coding agent 13 tools to query its own telemetry mid-session — just ask "how much have I spent today?" or "are there any active alerts?"
|
|
115
|
+
|
|
116
|
+
[Full Claude Code & Codex setup →](#claude-code--coding-agents)
|
|
117
|
+
|
|
118
|
+
### Python SDK
|
|
119
|
+
|
|
120
|
+
For any Python agent — Anthropic, OpenAI, Gemini, Bedrock, LangChain, CrewAI, and [10+ more](#supported-frameworks).
|
|
121
|
+
|
|
122
|
+
```bash
|
|
123
|
+
pip install tokenjam
|
|
124
|
+
tj onboard # creates config, generates ingest secret
|
|
125
|
+
ocw doctor # verify your setup
|
|
126
|
+
```
|
|
127
|
+
|
|
128
|
+
```python
|
|
129
|
+
from tokenjam.sdk import watch
|
|
130
|
+
from tokenjam.sdk.integrations.anthropic import patch_anthropic
|
|
131
|
+
|
|
132
|
+
patch_anthropic() # auto-intercepts all Anthropic API calls
|
|
133
|
+
|
|
134
|
+
@watch(agent_id="my-agent")
|
|
135
|
+
def run(task: str) -> str:
|
|
136
|
+
# your agent code — nothing else to change
|
|
137
|
+
...
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
One-line patches exist for every major provider and framework. [See all integrations →](#supported-frameworks)
|
|
141
|
+
|
|
142
|
+
### TypeScript SDK
|
|
143
|
+
|
|
144
|
+
For any Node.js / TypeScript agent. Sends spans to `tj serve` over HTTP.
|
|
145
|
+
|
|
146
|
+
```bash
|
|
147
|
+
npm install @tokenjam/sdk
|
|
148
|
+
```
|
|
149
|
+
|
|
150
|
+
```typescript
|
|
151
|
+
import { TjClient, SpanBuilder } from "@tokenjam/sdk";
|
|
152
|
+
|
|
153
|
+
const client = new TjClient({
|
|
154
|
+
baseUrl: "http://127.0.0.1:7391",
|
|
155
|
+
ingestSecret: process.env.TJ_INGEST_SECRET ?? "",
|
|
156
|
+
});
|
|
157
|
+
|
|
158
|
+
const span = new SpanBuilder("invoke_agent")
|
|
159
|
+
.agentId("my-ts-agent")
|
|
160
|
+
.model("gpt-4o-mini")
|
|
161
|
+
.provider("openai")
|
|
162
|
+
.inputTokens(450)
|
|
163
|
+
.outputTokens(120)
|
|
164
|
+
.build();
|
|
165
|
+
|
|
166
|
+
await client.send([span]);
|
|
167
|
+
```
|
|
168
|
+
|
|
169
|
+
### Any OTel-compatible agent
|
|
170
|
+
|
|
171
|
+
Already emitting OpenTelemetry? Point your OTLP exporter at `tj serve` — no SDK needed:
|
|
172
|
+
|
|
173
|
+
```bash
|
|
174
|
+
tj serve &
|
|
175
|
+
export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:7391
|
|
176
|
+
# run your agent as usual
|
|
177
|
+
```
|
|
178
|
+
|
|
179
|
+
| Framework | OTel support |
|
|
180
|
+
|---|---|
|
|
181
|
+
| **Claude Code** | Built-in — `tj onboard --claude-code` |
|
|
182
|
+
| **OpenClaw** | Built-in (`diagnostics-otel` plugin) — [setup guide](docs/openclaw.md) |
|
|
183
|
+
| LlamaIndex | `opentelemetry-instrumentation-llama-index` |
|
|
184
|
+
| OpenAI Agents SDK | Built-in |
|
|
185
|
+
| Google ADK | Built-in |
|
|
186
|
+
| Strands Agent SDK (AWS) | Built-in |
|
|
187
|
+
| Haystack | Built-in |
|
|
188
|
+
| Pydantic AI | Built-in |
|
|
189
|
+
| Semantic Kernel | Built-in |
|
|
190
|
+
|
|
191
|
+
---
|
|
192
|
+
|
|
193
|
+
## CLI
|
|
194
|
+
|
|
195
|
+
```
|
|
196
|
+
tj status
|
|
197
|
+
```
|
|
198
|
+
|
|
199
|
+
```
|
|
200
|
+
● my-email-agent completed (2m 14s)
|
|
201
|
+
|
|
202
|
+
Cost today: $0.0340 / $5.0000 limit
|
|
203
|
+
Tokens: 12.4k in / 3.8k out
|
|
204
|
+
Tool calls: 47
|
|
205
|
+
Active session: sess-a1b2c3
|
|
206
|
+
|
|
207
|
+
send_email called (sensitive action: critical)
|
|
208
|
+
```
|
|
209
|
+
|
|
210
|
+
https://github.com/user-attachments/assets/b94d13f6-1432-40d4-b093-6958d74f0e65
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
tj status # current state, cost, active alerts
|
|
214
|
+
ocw traces # full span history with waterfall view
|
|
215
|
+
ocw cost --since 7d # cost breakdown by agent, model, day
|
|
216
|
+
ocw alerts # everything that fired while you were away
|
|
217
|
+
ocw budget # view and set daily/session cost limits
|
|
218
|
+
ocw drift # behavioral drift Z-scores vs baseline
|
|
219
|
+
ocw tools # tool call history with error rates
|
|
220
|
+
tj serve # start the web UI + REST API
|
|
221
|
+
```
|
|
222
|
+
|
|
223
|
+
---
|
|
224
|
+
|
|
225
|
+
## Web UI
|
|
226
|
+
|
|
227
|
+
`tj serve` starts a local dashboard at `http://127.0.0.1:7391/`.
|
|
228
|
+
|
|
229
|
+
https://github.com/user-attachments/assets/ff09caec-3487-4542-8628-d62b7d92591f
|
|
230
|
+
|
|
231
|
+
- **Status** — agent overview with cost, tokens, tool calls, and active alerts
|
|
232
|
+
- **Traces** — trace list with span waterfall visualization
|
|
233
|
+
- **Cost** — breakdown by agent, model, day, or tool
|
|
234
|
+
- **Alerts** — alert history with severity filtering
|
|
235
|
+
- **Budget** — view and edit daily/session cost limits per agent
|
|
236
|
+
- **Drift** — behavioral drift report with Z-score analysis
|
|
237
|
+
|
|
238
|
+
No signup, no cloud — runs entirely on your machine.
|
|
239
|
+
|
|
240
|
+
---
|
|
241
|
+
|
|
242
|
+
## ocw vs LangSmith vs Langfuse
|
|
243
|
+
|
|
244
|
+
LangSmith and Langfuse are excellent for tracing LLM API calls and running evals on chat outputs. `tj` solves a different problem: **autonomous agents running unsupervised with real-world consequences**.
|
|
245
|
+
|
|
246
|
+
| | `tj` | LangSmith | Langfuse | Datadog LLM Obs |
|
|
247
|
+
|---|---|---|---|---|
|
|
248
|
+
| Signup required | ❌ | ✅ | ✅ | ✅ |
|
|
249
|
+
| Data leaves your machine | ❌ | ✅ | cloud only | ✅ |
|
|
250
|
+
| Real-time sensitive action alerts | ✅ | ❌ | ❌ | ❌ |
|
|
251
|
+
| Behavioral drift detection | ✅ | ❌ | ❌ | ❌ |
|
|
252
|
+
| Local-first, no cloud required | ✅ | ❌ | self-host only | ❌ |
|
|
253
|
+
| OTel GenAI SemConv native | ✅ | partial | partial | partial |
|
|
254
|
+
| NemoClaw sandbox events | ✅ | ❌ | ❌ | ❌ |
|
|
255
|
+
| Works with any agent/framework | ✅ | LangChain-first | partial | ❌ |
|
|
256
|
+
| Free, MIT licensed | ✅ | freemium | freemium | paid |
|
|
257
|
+
|
|
258
|
+
---
|
|
259
|
+
|
|
260
|
+
## Claude Code + coding agents
|
|
261
|
+
|
|
262
|
+
### Claude Code
|
|
263
|
+
|
|
264
|
+
Monitor every Claude Code session — costs, tool calls, API requests, errors — with two commands:
|
|
265
|
+
|
|
266
|
+
```bash
|
|
267
|
+
pip install "tokenjam[mcp]"
|
|
268
|
+
tj onboard --claude-code
|
|
269
|
+
# Restart Claude Code, then:
|
|
270
|
+
tj status --agent claude-code-<project>
|
|
271
|
+
```
|
|
272
|
+
|
|
273
|
+
`tj onboard --claude-code` does everything in one shot:
|
|
274
|
+
- Creates a shared config at `~/.config/tj/config.toml` (one config for all projects)
|
|
275
|
+
- Writes OTLP exporter vars to `~/.claude/settings.json`
|
|
276
|
+
- Tags this project by writing `OTEL_RESOURCE_ATTRIBUTES` to `.claude/settings.json`
|
|
277
|
+
- Registers the MCP server globally (`claude mcp add --scope user tj -- tj mcp`)
|
|
278
|
+
- Installs a background daemon (launchd on macOS, systemd on Linux)
|
|
279
|
+
- Adds Docker harness-compatible OTLP env vars to `~/.zshrc`
|
|
280
|
+
|
|
281
|
+
**Claude Code must be restarted** after running `tj onboard --claude-code`.
|
|
282
|
+
|
|
283
|
+
**Adding more projects** — run once per project directory:
|
|
284
|
+
|
|
285
|
+
```bash
|
|
286
|
+
cd /path/to/other-project
|
|
287
|
+
tj onboard --claude-code # tags this project, no reinstall needed
|
|
288
|
+
# Restart Claude Code
|
|
289
|
+
```
|
|
290
|
+
|
|
291
|
+
Each project gets its own agent ID (`claude-code-<repo-name>`), all sharing one server and one ingest secret.
|
|
292
|
+
|
|
293
|
+
### MCP server
|
|
294
|
+
|
|
295
|
+
The MCP server gives Claude Code direct access to your observability data inside the session. 13 tools available after restart:
|
|
296
|
+
|
|
297
|
+
| Tool | What it does |
|
|
298
|
+
|---|---|
|
|
299
|
+
| `get_status` | Current agent state — tokens, cost, active alerts |
|
|
300
|
+
| `get_budget_headroom` | Budget limit vs spend |
|
|
301
|
+
| `list_active_sessions` | All running sessions across agents |
|
|
302
|
+
| `list_agents` | All known agents with lifetime cost |
|
|
303
|
+
| `get_cost_summary` | Cost breakdown by day / agent / model |
|
|
304
|
+
| `list_alerts` | Alert history with severity filtering |
|
|
305
|
+
| `list_traces` | Recent traces with cost and duration |
|
|
306
|
+
| `get_trace` | Full span waterfall for a trace |
|
|
307
|
+
| `get_tool_stats` | Tool call counts and average duration |
|
|
308
|
+
| `get_drift_report` | Drift baseline vs latest session |
|
|
309
|
+
| `acknowledge_alert` | Mark an alert as acknowledged |
|
|
310
|
+
| `setup_project` | Configure a project for OCW telemetry |
|
|
311
|
+
| `open_dashboard` | Open the web UI (starts `tj serve` if needed) |
|
|
312
|
+
|
|
313
|
+
The MCP server opens DuckDB read-only — no lock conflicts with `tj serve`.
|
|
314
|
+
|
|
315
|
+
**Per-project tagging** — after installing globally, ask Claude Code:
|
|
316
|
+
|
|
317
|
+
> "Set up OCW for this project"
|
|
318
|
+
|
|
319
|
+
Claude calls `setup_project`, which writes `.claude/settings.json` with the right `OTEL_RESOURCE_ATTRIBUTES` for this project.
|
|
320
|
+
|
|
321
|
+
### Codex
|
|
322
|
+
|
|
323
|
+
Monitor every Codex session — run once, globally:
|
|
324
|
+
|
|
325
|
+
```bash
|
|
326
|
+
pip install "tokenjam[mcp]"
|
|
327
|
+
tj onboard --codex
|
|
328
|
+
```
|
|
329
|
+
|
|
330
|
+
`tj onboard --codex` is project-agnostic. It writes to `~/.codex/config.toml` (Codex's single global config), so you only run it once — not once per project. Codex hardcodes `service.name="codex_exec"` in its binary, so all sessions appear under the same agent ID regardless of which repo you're working in.
|
|
331
|
+
|
|
332
|
+
`tj onboard --codex`:
|
|
333
|
+
- Writes an `[otel]` block and `[mcp_servers.ocw]` to `~/.codex/config.toml`
|
|
334
|
+
- Registers the MCP server so Codex can call OCW tools directly
|
|
335
|
+
- Installs the background daemon (launchd / systemd)
|
|
336
|
+
|
|
337
|
+
**Codex must be restarted** after running `tj onboard --codex`.
|
|
338
|
+
|
|
339
|
+
```bash
|
|
340
|
+
tj status --agent codex_exec # check it's working
|
|
341
|
+
```
|
|
342
|
+
|
|
343
|
+
The same 13 MCP tools available to Claude Code are available to Codex after restart.
|
|
344
|
+
|
|
345
|
+
### Uninstalling
|
|
346
|
+
|
|
347
|
+
```bash
|
|
348
|
+
# Remove all OCW data, config, daemon, MCP registration, and env vars:
|
|
349
|
+
ocw uninstall --yes
|
|
350
|
+
|
|
351
|
+
# Then remove the package:
|
|
352
|
+
pip uninstall tokenjam -y
|
|
353
|
+
```
|
|
354
|
+
|
|
355
|
+
`tj uninstall` cleans up everything set by `tj onboard --claude-code`: daemon, MCP server, `~/.ocw/`, `~/.config/ocw/`, OTLP env vars in `~/.claude/settings.json`, `OTEL_RESOURCE_ATTRIBUTES` in every onboarded project's `.claude/settings.json`, and the harness env block in `~/.zshrc`.
|
|
356
|
+
|
|
357
|
+
---
|
|
358
|
+
|
|
359
|
+
## Supported frameworks
|
|
360
|
+
|
|
361
|
+
### Python — provider patches
|
|
362
|
+
|
|
363
|
+
Intercept at the API level. Framework-agnostic.
|
|
364
|
+
|
|
365
|
+
```python
|
|
366
|
+
from tokenjam.sdk.integrations.anthropic import patch_anthropic # Anthropic
|
|
367
|
+
from tokenjam.sdk.integrations.openai import patch_openai # OpenAI
|
|
368
|
+
from tokenjam.sdk.integrations.gemini import patch_gemini # Google Gemini
|
|
369
|
+
from tokenjam.sdk.integrations.bedrock import patch_bedrock # AWS Bedrock
|
|
370
|
+
from tokenjam.sdk.integrations.litellm import patch_litellm # LiteLLM (100+ providers)
|
|
371
|
+
```
|
|
372
|
+
|
|
373
|
+
`patch_litellm()` covers all providers LiteLLM routes to (OpenAI, Anthropic, Bedrock, Vertex, Cohere, Mistral, Ollama, etc.). If you use LiteLLM, you don't need individual patches.
|
|
374
|
+
|
|
375
|
+
OpenAI-compatible providers (Groq, Together, Fireworks, xAI, Azure OpenAI) work via `patch_openai(base_url=...)`.
|
|
376
|
+
|
|
377
|
+
### Python — framework patches
|
|
378
|
+
|
|
379
|
+
Instrument the framework's own abstractions:
|
|
380
|
+
|
|
381
|
+
```python
|
|
382
|
+
from tokenjam.sdk.integrations.langchain import patch_langchain # BaseLLM + BaseTool
|
|
383
|
+
from tokenjam.sdk.integrations.langgraph import patch_langgraph # CompiledGraph
|
|
384
|
+
from tokenjam.sdk.integrations.crewai import patch_crewai # Task + Agent
|
|
385
|
+
from tokenjam.sdk.integrations.autogen import patch_autogen # ConversableAgent
|
|
386
|
+
from tokenjam.sdk.integrations.llamaindex import patch_llamaindex # Native OTel
|
|
387
|
+
from tokenjam.sdk.integrations.openai_agents_sdk import patch_openai_agents # Native OTel
|
|
388
|
+
from tokenjam.sdk.integrations.nemoclaw import watch_nemoclaw # NemoClaw Gateway
|
|
389
|
+
```
|
|
390
|
+
|
|
391
|
+
Full framework support guide: [docs/framework-support.md](docs/framework-support.md)
|
|
392
|
+
|
|
393
|
+
---
|
|
394
|
+
|
|
395
|
+
## Alert channels
|
|
396
|
+
|
|
397
|
+
Configure where alerts go. Multiple channels work simultaneously.
|
|
398
|
+
|
|
399
|
+
```toml
|
|
400
|
+
# .ocw/config.toml
|
|
401
|
+
|
|
402
|
+
[[alerts.channels]]
|
|
403
|
+
type = "ntfy"
|
|
404
|
+
topic = "my-agent-alerts" # push to your phone, free, no account
|
|
405
|
+
|
|
406
|
+
[[alerts.channels]]
|
|
407
|
+
type = "discord"
|
|
408
|
+
webhook_url = "https://discord.com/api/webhooks/..."
|
|
409
|
+
|
|
410
|
+
[[alerts.channels]]
|
|
411
|
+
type = "webhook"
|
|
412
|
+
url = "https://your-endpoint.com/alerts"
|
|
413
|
+
```
|
|
414
|
+
|
|
415
|
+
Alert types: `sensitive_action` · `cost_budget_daily` · `cost_budget_session` · `session_duration` · `retry_loop` · `token_anomaly` · `schema_violation` · `drift_detected` · `failure_rate` · `network_egress_blocked` · `filesystem_access_denied` · `syscall_denied` · `inference_rerouted`
|
|
416
|
+
|
|
417
|
+
Full alert reference — trigger conditions, cooldown config, content stripping, all 6 channel types: [docs/alerts.md](docs/alerts.md)
|
|
418
|
+
|
|
419
|
+
---
|
|
420
|
+
|
|
421
|
+
## NemoClaw support
|
|
422
|
+
|
|
423
|
+
Running OpenClaw inside [NVIDIA NemoClaw](https://github.com/NVIDIA/NemoClaw)? `tj` connects to the OpenShell Gateway WebSocket and turns sandbox events — blocked network requests, filesystem denials, inference reroutes — into alerts.
|
|
424
|
+
|
|
425
|
+
```python
|
|
426
|
+
from tokenjam.sdk.integrations.nemoclaw import watch_nemoclaw
|
|
427
|
+
|
|
428
|
+
observer = watch_nemoclaw()
|
|
429
|
+
asyncio.create_task(observer.connect())
|
|
430
|
+
```
|
|
431
|
+
|
|
432
|
+
This is the observability layer that NemoClaw doesn't ship with.
|
|
433
|
+
|
|
434
|
+
Full event table and configuration: [docs/nemoclaw-integration.md](docs/nemoclaw-integration.md)
|
|
435
|
+
|
|
436
|
+
---
|
|
437
|
+
|
|
438
|
+
## Export and integrate
|
|
439
|
+
|
|
440
|
+
```bash
|
|
441
|
+
ocw export --format otlp # forward to Grafana, Datadog, any OTel backend
|
|
442
|
+
ocw export --format openevals # openevals / agentevals trajectory evaluation
|
|
443
|
+
ocw export --format json # NDJSON
|
|
444
|
+
ocw export --format csv
|
|
445
|
+
```
|
|
446
|
+
|
|
447
|
+
Prometheus metrics at `http://127.0.0.1:7391/metrics` when `tj serve` is running.
|
|
448
|
+
|
|
449
|
+
Export filtering flags, REST API endpoints, and API docs: [docs/export.md](docs/export.md)
|
|
450
|
+
|
|
451
|
+
---
|
|
452
|
+
|
|
453
|
+
## Architecture
|
|
454
|
+
|
|
455
|
+
```mermaid
|
|
456
|
+
flowchart TD
|
|
457
|
+
Agent["Your agent"]
|
|
458
|
+
|
|
459
|
+
Agent --> Terminal["Coding agents\nClaude Code · Codex"]
|
|
460
|
+
Agent --> PythonSDK["Python SDK\n@watch + patch_*"]
|
|
461
|
+
Agent --> TypeScriptSDK["TypeScript SDK\n@tokenjam/sdk"]
|
|
462
|
+
|
|
463
|
+
Terminal --> OTLP["OTLP export"]
|
|
464
|
+
PythonSDK --> Exporter["TjSpanExporter"]
|
|
465
|
+
TypeScriptSDK --> HTTP["POST /api/v1/spans"]
|
|
466
|
+
OTLP --> HTTP
|
|
467
|
+
|
|
468
|
+
Exporter --> Ingest
|
|
469
|
+
HTTP --> Ingest
|
|
470
|
+
|
|
471
|
+
Ingest["IngestPipeline\nSanitize · Session continuity · Extract"]
|
|
472
|
+
|
|
473
|
+
Ingest --> Cost["CostEngine\npricing.toml"]
|
|
474
|
+
Ingest --> Alerts["AlertEngine\n13 types · 6 channels"]
|
|
475
|
+
Ingest --> Schema["SchemaValidator\nJSON Schema + infer"]
|
|
476
|
+
|
|
477
|
+
Cost --> DB["DuckDB\nlocal · embedded"]
|
|
478
|
+
Alerts --> DB
|
|
479
|
+
Schema --> DB
|
|
480
|
+
|
|
481
|
+
DB --> CLI["ocw CLI"]
|
|
482
|
+
DB --> API["REST API + Web UI\n:7391"]
|
|
483
|
+
DB --> MCP["MCP Server\n13 tools"]
|
|
484
|
+
DB --> Prom["Prometheus\n:7391/metrics"]
|
|
485
|
+
```
|
|
486
|
+
|
|
487
|
+
Full architecture deep-dive — design principles, SDK internals, alert system, testing: [docs/architecture.md](docs/architecture.md)
|
|
488
|
+
|
|
489
|
+
---
|
|
490
|
+
|
|
491
|
+
## Configuration
|
|
492
|
+
|
|
493
|
+
```toml
|
|
494
|
+
# .ocw/config.toml — generated by tj onboard
|
|
495
|
+
|
|
496
|
+
[defaults.budget]
|
|
497
|
+
daily_usd = 10.00
|
|
498
|
+
|
|
499
|
+
[agents.my-email-agent]
|
|
500
|
+
description = "Personal email management agent"
|
|
501
|
+
|
|
502
|
+
[agents.my-email-agent.budget]
|
|
503
|
+
daily_usd = 5.00
|
|
504
|
+
session_usd = 1.00
|
|
505
|
+
|
|
506
|
+
[[agents.my-email-agent.sensitive_actions]]
|
|
507
|
+
name = "send_email"
|
|
508
|
+
severity = "critical"
|
|
509
|
+
|
|
510
|
+
[agents.my-email-agent.drift]
|
|
511
|
+
enabled = true
|
|
512
|
+
baseline_sessions = 10
|
|
513
|
+
token_threshold = 2.0
|
|
514
|
+
|
|
515
|
+
[capture]
|
|
516
|
+
prompts = false
|
|
517
|
+
completions = false
|
|
518
|
+
tool_outputs = false
|
|
519
|
+
|
|
520
|
+
[storage]
|
|
521
|
+
path = "~/.ocw/telemetry.duckdb"
|
|
522
|
+
retention_days = 90
|
|
523
|
+
```
|
|
524
|
+
|
|
525
|
+
Budget limits merge per-field: each agent inherits defaults unless overridden. Set via CLI (`tj budget --daily 10`), API, or web UI. Run `tj doctor` to verify.
|
|
526
|
+
|
|
527
|
+
Config file discovery order, full config schema, API auth, capture settings: [docs/configuration.md](docs/configuration.md)
|
|
528
|
+
|
|
529
|
+
---
|
|
530
|
+
|
|
531
|
+
## CLI reference
|
|
532
|
+
|
|
533
|
+
16 commands: `onboard`, `doctor`, `status`, `traces`, `cost`, `alerts`, `budget`, `drift`, `tools`, `demo`, `export`, `mcp`, `serve`, `stop`, `uninstall`. All support `--json` for machine-readable output.
|
|
534
|
+
|
|
535
|
+
Global flags, per-command options, exit codes: [docs/cli-reference.md](docs/cli-reference.md)
|
|
536
|
+
|
|
537
|
+
---
|
|
538
|
+
|
|
539
|
+
## Examples
|
|
540
|
+
|
|
541
|
+
The [`examples/`](examples/) directory has runnable agents for every integration:
|
|
542
|
+
|
|
543
|
+
- **Single provider** — Anthropic, OpenAI, Gemini, Bedrock, OpenAI Agents SDK
|
|
544
|
+
- **Single framework** — LangChain, LangGraph, CrewAI, AutoGen, LlamaIndex
|
|
545
|
+
- **Multi-integration** — provider router, CrewAI + LangChain, RAG with fallback
|
|
546
|
+
- **Alerts and drift** — sensitive action alerts, budget breach, drift detection (no API keys needed)
|
|
547
|
+
|
|
548
|
+
```bash
|
|
549
|
+
python examples/single_provider/anthropic_agent.py
|
|
550
|
+
python examples/alerts_and_drift/drift_demo.py # no API key needed
|
|
551
|
+
```
|
|
552
|
+
|
|
553
|
+
See [`examples/README.md`](examples/README.md) for the full list.
|
|
554
|
+
|
|
555
|
+
---
|
|
556
|
+
|
|
557
|
+
## Agent Incident Library
|
|
558
|
+
|
|
559
|
+
Reproducible AI agent failures you can run in 30 seconds. No API keys, no config, no setup.
|
|
560
|
+
|
|
561
|
+
```bash
|
|
562
|
+
ocw demo # list all scenarios
|
|
563
|
+
ocw demo retry-loop # run one
|
|
564
|
+
ocw demo retry-loop --json # machine-readable output
|
|
565
|
+
```
|
|
566
|
+
|
|
567
|
+
| Scenario | What goes wrong | What OCW catches |
|
|
568
|
+
|---|---|---|
|
|
569
|
+
| [`retry-loop`](incidents/retry-loop/README.md) | Agent retries a failing tool in a loop, burning time and tokens | `retry_loop` + `failure_rate` alerts fire automatically |
|
|
570
|
+
| [`surprise-cost`](incidents/surprise-cost/README.md) | Model silently escalates from Haiku to Opus mid-chain | Per-model cost breakdown shows the $3+ you didn't expect |
|
|
571
|
+
| [`hallucination-drift`](incidents/hallucination-drift/README.md) | Agent behavior shifts — different tokens, different tools | `drift_detected` alert fires with Z-scores at session end |
|
|
572
|
+
|
|
573
|
+
Each scenario runs against an in-memory backend and produces a side-by-side comparison: what `print()` shows vs. what OCW reveals.
|
|
574
|
+
|
|
575
|
+
---
|
|
576
|
+
|
|
577
|
+
## Architecture
|
|
578
|
+
|
|
579
|
+
See [AGENTS.md](AGENTS.md) for codebase conventions.
|
|
580
|
+
|
|
581
|
+
PRs welcome. If you're adding a framework integration, open an issue first.
|
|
582
|
+
|
|
583
|
+
---
|
|
584
|
+
|
|
585
|
+
## Roadmap
|
|
586
|
+
|
|
587
|
+
**Shipped:**
|
|
588
|
+
|
|
589
|
+
- [x] `tj serve` background daemon (launchd / systemd)
|
|
590
|
+
- [x] Web UI with auto-polling (status, traces, cost, alerts, budget, drift)
|
|
591
|
+
- [x] LiteLLM provider patch (100+ providers)
|
|
592
|
+
- [x] `tj stop` and `tj uninstall`
|
|
593
|
+
- [x] Claude Code integration (`tj onboard --claude-code`)
|
|
594
|
+
- [x] Codex integration (`tj onboard --codex`)
|
|
595
|
+
- [x] OpenClaw integration (zero-code via `diagnostics-otel` plugin)
|
|
596
|
+
- [x] NemoClaw sandbox observer (WebSocket gateway events)
|
|
597
|
+
- [x] OTLP log-to-span pipeline (Claude Code log events)
|
|
598
|
+
- [x] `tj budget` CLI, API, and web UI
|
|
599
|
+
- [x] `tj drift` with Z-score reporting
|
|
600
|
+
- [x] Full pipeline wiring (alerts, schema, drift in `tj serve`)
|
|
601
|
+
- [x] MCP server — 13 tools for Claude Code
|
|
602
|
+
|
|
603
|
+
**Up next:**
|
|
604
|
+
|
|
605
|
+
- [ ] `tj watch` — live tail mode for spans
|
|
606
|
+
- [ ] `tj replay` — replay captured sessions against new model versions
|
|
607
|
+
- [ ] TypeScript framework patches (LangChain JS, OpenAI Agents SDK)
|
|
608
|
+
- [ ] Vercel AI SDK integration (TypeScript)
|
|
609
|
+
- [ ] Mastra integration (TypeScript)
|
|
610
|
+
- [ ] Azure AI Agent Service integration
|
|
611
|
+
- [ ] Docker image
|
|
612
|
+
- [ ] GitHub Actions for CI drift/cost checks
|
|
613
|
+
|
|
614
|
+
---
|
|
615
|
+
|
|
616
|
+
<div align="center">
|
|
617
|
+
|
|
618
|
+
**[opencla.watch](https://opencla.watch)** · [PyPI](https://pypi.org/project/tokenjam/) · [npm](https://www.npmjs.com/package/@tokenjam/sdk)
|
|
619
|
+
|
|
620
|
+
MIT License · Built by [Metabuilder Labs](https://github.com/Metabuilder-Labs)
|
|
621
|
+
|
|
622
|
+
</div>
|