tokenjam 0.2.0__py3-none-any.whl

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (86) hide show
  1. tokenjam/__init__.py +1 -0
  2. tokenjam/api/__init__.py +0 -0
  3. tokenjam/api/app.py +104 -0
  4. tokenjam/api/deps.py +18 -0
  5. tokenjam/api/middleware.py +28 -0
  6. tokenjam/api/routes/__init__.py +0 -0
  7. tokenjam/api/routes/agents.py +33 -0
  8. tokenjam/api/routes/alerts.py +77 -0
  9. tokenjam/api/routes/budget.py +96 -0
  10. tokenjam/api/routes/cost.py +43 -0
  11. tokenjam/api/routes/drift.py +63 -0
  12. tokenjam/api/routes/logs.py +511 -0
  13. tokenjam/api/routes/metrics.py +81 -0
  14. tokenjam/api/routes/otlp.py +63 -0
  15. tokenjam/api/routes/spans.py +202 -0
  16. tokenjam/api/routes/status.py +84 -0
  17. tokenjam/api/routes/tools.py +22 -0
  18. tokenjam/api/routes/traces.py +92 -0
  19. tokenjam/cli/__init__.py +0 -0
  20. tokenjam/cli/cmd_alerts.py +94 -0
  21. tokenjam/cli/cmd_budget.py +119 -0
  22. tokenjam/cli/cmd_cost.py +90 -0
  23. tokenjam/cli/cmd_demo.py +82 -0
  24. tokenjam/cli/cmd_doctor.py +173 -0
  25. tokenjam/cli/cmd_drift.py +238 -0
  26. tokenjam/cli/cmd_export.py +200 -0
  27. tokenjam/cli/cmd_mcp.py +78 -0
  28. tokenjam/cli/cmd_onboard.py +779 -0
  29. tokenjam/cli/cmd_serve.py +85 -0
  30. tokenjam/cli/cmd_status.py +153 -0
  31. tokenjam/cli/cmd_stop.py +87 -0
  32. tokenjam/cli/cmd_tools.py +45 -0
  33. tokenjam/cli/cmd_traces.py +161 -0
  34. tokenjam/cli/cmd_uninstall.py +159 -0
  35. tokenjam/cli/main.py +110 -0
  36. tokenjam/core/__init__.py +0 -0
  37. tokenjam/core/alerts.py +619 -0
  38. tokenjam/core/api_backend.py +235 -0
  39. tokenjam/core/config.py +360 -0
  40. tokenjam/core/cost.py +102 -0
  41. tokenjam/core/db.py +718 -0
  42. tokenjam/core/drift.py +256 -0
  43. tokenjam/core/ingest.py +265 -0
  44. tokenjam/core/models.py +225 -0
  45. tokenjam/core/pricing.py +54 -0
  46. tokenjam/core/retention.py +21 -0
  47. tokenjam/core/schema_validator.py +156 -0
  48. tokenjam/demo/__init__.py +0 -0
  49. tokenjam/demo/env.py +96 -0
  50. tokenjam/mcp/__init__.py +0 -0
  51. tokenjam/mcp/server.py +1067 -0
  52. tokenjam/otel/__init__.py +0 -0
  53. tokenjam/otel/exporters.py +26 -0
  54. tokenjam/otel/provider.py +207 -0
  55. tokenjam/otel/semconv.py +144 -0
  56. tokenjam/pricing/models.toml +70 -0
  57. tokenjam/py.typed +0 -0
  58. tokenjam/sdk/__init__.py +21 -0
  59. tokenjam/sdk/agent.py +206 -0
  60. tokenjam/sdk/bootstrap.py +120 -0
  61. tokenjam/sdk/http_exporter.py +109 -0
  62. tokenjam/sdk/integrations/__init__.py +0 -0
  63. tokenjam/sdk/integrations/anthropic.py +200 -0
  64. tokenjam/sdk/integrations/autogen.py +97 -0
  65. tokenjam/sdk/integrations/base.py +27 -0
  66. tokenjam/sdk/integrations/bedrock.py +103 -0
  67. tokenjam/sdk/integrations/crewai.py +96 -0
  68. tokenjam/sdk/integrations/gemini.py +131 -0
  69. tokenjam/sdk/integrations/langchain.py +156 -0
  70. tokenjam/sdk/integrations/langgraph.py +101 -0
  71. tokenjam/sdk/integrations/litellm.py +323 -0
  72. tokenjam/sdk/integrations/llamaindex.py +52 -0
  73. tokenjam/sdk/integrations/nemoclaw.py +139 -0
  74. tokenjam/sdk/integrations/openai.py +159 -0
  75. tokenjam/sdk/integrations/openai_agents_sdk.py +47 -0
  76. tokenjam/sdk/transport.py +98 -0
  77. tokenjam/ui/index.html +1213 -0
  78. tokenjam/utils/__init__.py +0 -0
  79. tokenjam/utils/formatting.py +43 -0
  80. tokenjam/utils/ids.py +15 -0
  81. tokenjam/utils/time_parse.py +54 -0
  82. tokenjam-0.2.0.dist-info/METADATA +622 -0
  83. tokenjam-0.2.0.dist-info/RECORD +86 -0
  84. tokenjam-0.2.0.dist-info/WHEEL +4 -0
  85. tokenjam-0.2.0.dist-info/entry_points.txt +2 -0
  86. tokenjam-0.2.0.dist-info/licenses/LICENSE +21 -0
@@ -0,0 +1,622 @@
1
+ Metadata-Version: 2.4
2
+ Name: tokenjam
3
+ Version: 0.2.0
4
+ Summary: TokenJam — local-first OTel-native observability for Autonomous AI agents
5
+ Project-URL: Homepage, https://opencla.watch
6
+ Project-URL: Repository, https://github.com/Metabuilder-Labs/openclawwatch
7
+ Project-URL: Issues, https://github.com/Metabuilder-Labs/openclawwatch/issues
8
+ Author-email: Anil Murty <anil@metabldr.com>
9
+ License: MIT
10
+ License-File: LICENSE
11
+ Keywords: agents,ai,llm,observability,opentelemetry
12
+ Classifier: Development Status :: 4 - Beta
13
+ Classifier: Intended Audience :: Developers
14
+ Classifier: License :: OSI Approved :: MIT License
15
+ Classifier: Programming Language :: Python :: 3
16
+ Classifier: Programming Language :: Python :: 3.10
17
+ Classifier: Programming Language :: Python :: 3.11
18
+ Classifier: Programming Language :: Python :: 3.12
19
+ Classifier: Topic :: Software Development :: Libraries
20
+ Classifier: Topic :: System :: Monitoring
21
+ Requires-Python: >=3.10
22
+ Requires-Dist: apscheduler>=3.10
23
+ Requires-Dist: click>=8.1
24
+ Requires-Dist: duckdb>=0.10
25
+ Requires-Dist: fastapi>=0.110
26
+ Requires-Dist: genson>=1.2
27
+ Requires-Dist: httpx>=0.27
28
+ Requires-Dist: jsonschema>=4.0
29
+ Requires-Dist: opentelemetry-exporter-otlp-proto-grpc>=1.25
30
+ Requires-Dist: opentelemetry-exporter-otlp-proto-http>=1.25
31
+ Requires-Dist: opentelemetry-exporter-prometheus>=0.46b0
32
+ Requires-Dist: opentelemetry-sdk>=1.25
33
+ Requires-Dist: pytz>=2024.1
34
+ Requires-Dist: rich>=13.0
35
+ Requires-Dist: tomli-w>=1.0
36
+ Requires-Dist: tomli>=2.0; python_version < '3.11'
37
+ Requires-Dist: uvicorn>=0.27
38
+ Requires-Dist: websockets>=12.0
39
+ Provides-Extra: autogen
40
+ Requires-Dist: pyautogen>=0.2; extra == 'autogen'
41
+ Provides-Extra: crewai
42
+ Requires-Dist: crewai>=0.28; extra == 'crewai'
43
+ Provides-Extra: dev
44
+ Requires-Dist: httpx; extra == 'dev'
45
+ Requires-Dist: mypy; extra == 'dev'
46
+ Requires-Dist: pytest; extra == 'dev'
47
+ Requires-Dist: pytest-asyncio; extra == 'dev'
48
+ Requires-Dist: ruff; extra == 'dev'
49
+ Provides-Extra: langchain
50
+ Requires-Dist: langchain>=0.2; extra == 'langchain'
51
+ Provides-Extra: litellm
52
+ Requires-Dist: litellm>=1.40; extra == 'litellm'
53
+ Provides-Extra: mcp
54
+ Requires-Dist: fastmcp; extra == 'mcp'
55
+ Description-Content-Type: text/markdown
56
+
57
+ <div align="center">
58
+
59
+ <img src="https://opencla.watch/icon.svg" alt="TokenJam" width="72" height="72">
60
+
61
+ # TokenJam
62
+
63
+ The open-source LLM observability tool for autonomous agents.
64
+
65
+ No cloud. No signup. No surprises.
66
+
67
+ [![CI](https://github.com/Metabuilder-Labs/tokenjam/actions/workflows/ci.yml/badge.svg)](https://github.com/Metabuilder-Labs/tokenjam/actions/workflows/ci.yml)
68
+ [![PyPI](https://img.shields.io/pypi/v/tokenjam?color=3d8eff&labelColor=0d1117)](https://pypi.org/project/tokenjam/)
69
+ [![Python](https://img.shields.io/badge/python-3.10%2B-3d8eff?labelColor=0d1117)](https://pypi.org/project/tokenjam/)
70
+ [![npm](https://img.shields.io/npm/v/@tokenjam/sdk?color=3d8eff&labelColor=0d1117)](https://www.npmjs.com/package/@tokenjam/sdk)
71
+ [![License: MIT](https://img.shields.io/badge/license-MIT-3d8eff?labelColor=0d1117)](LICENSE)
72
+ [![OTel](https://img.shields.io/badge/OTel-GenAI%20SemConv-3d8eff?labelColor=0d1117)](https://opentelemetry.io/docs/specs/semconv/gen-ai/)
73
+
74
+ ```
75
+ pip install tokenjam
76
+ ```
77
+
78
+ </div>
79
+
80
+ ---
81
+
82
+ Your agent sends emails, writes files, calls APIs, and spends your money — all while you're away. Most observability tools were built for LLM developers building chat products. `tj` was built for **agents with real-world consequences**: real-time cost tracking, safety alerts, behavioral drift detection, all running locally on your machine.
83
+
84
+ ---
85
+
86
+ ## What you get
87
+
88
+ **Real-time cost tracking.** Every LLM call is priced as it happens — by agent, model, session, and tool. Budget alerts fire before you hit the limit, not after.
89
+
90
+ **Safety alerts.** Configure any tool call as a sensitive action (`send_email`, `delete_file`, `submit_form`) and get notified instantly via ntfy, Discord, Telegram, webhook, or stdout.
91
+
92
+ **Behavioral drift detection.** `tj` builds a statistical baseline from your agent's real behavior and alerts when something deviates — a prompt tweak, a model update, a dependency bump. No LLM required.
93
+
94
+ **Tool output validation.** Declare a JSON Schema for your tools or let `tj` infer one automatically. Schema violations are caught the moment they occur.
95
+
96
+ **100% local.** DuckDB. Local REST API. No cloud backend. No API key for `tj` itself. Your telemetry never leaves your machine unless you explicitly export it.
97
+
98
+ ---
99
+
100
+ ## Get started
101
+
102
+ `tj` works four ways. Pick the one that fits.
103
+
104
+ ### Coding agents — zero code
105
+
106
+ For **Claude Code**, **Codex**, and any agent that already emits OpenTelemetry. No SDK, no code changes.
107
+
108
+ ```bash
109
+ pip install "tokenjam[mcp]"
110
+ tj onboard --claude-code # or: tj onboard --codex
111
+ # Restart your coding agent
112
+ ```
113
+
114
+ Every session, API call, tool use, and error is now a tracked span with cost and alert evaluation. The MCP server gives your coding agent 13 tools to query its own telemetry mid-session — just ask "how much have I spent today?" or "are there any active alerts?"
115
+
116
+ [Full Claude Code & Codex setup →](#claude-code--coding-agents)
117
+
118
+ ### Python SDK
119
+
120
+ For any Python agent — Anthropic, OpenAI, Gemini, Bedrock, LangChain, CrewAI, and [10+ more](#supported-frameworks).
121
+
122
+ ```bash
123
+ pip install tokenjam
124
+ tj onboard # creates config, generates ingest secret
125
+ ocw doctor # verify your setup
126
+ ```
127
+
128
+ ```python
129
+ from tokenjam.sdk import watch
130
+ from tokenjam.sdk.integrations.anthropic import patch_anthropic
131
+
132
+ patch_anthropic() # auto-intercepts all Anthropic API calls
133
+
134
+ @watch(agent_id="my-agent")
135
+ def run(task: str) -> str:
136
+ # your agent code — nothing else to change
137
+ ...
138
+ ```
139
+
140
+ One-line patches exist for every major provider and framework. [See all integrations →](#supported-frameworks)
141
+
142
+ ### TypeScript SDK
143
+
144
+ For any Node.js / TypeScript agent. Sends spans to `tj serve` over HTTP.
145
+
146
+ ```bash
147
+ npm install @tokenjam/sdk
148
+ ```
149
+
150
+ ```typescript
151
+ import { TjClient, SpanBuilder } from "@tokenjam/sdk";
152
+
153
+ const client = new TjClient({
154
+ baseUrl: "http://127.0.0.1:7391",
155
+ ingestSecret: process.env.TJ_INGEST_SECRET ?? "",
156
+ });
157
+
158
+ const span = new SpanBuilder("invoke_agent")
159
+ .agentId("my-ts-agent")
160
+ .model("gpt-4o-mini")
161
+ .provider("openai")
162
+ .inputTokens(450)
163
+ .outputTokens(120)
164
+ .build();
165
+
166
+ await client.send([span]);
167
+ ```
168
+
169
+ ### Any OTel-compatible agent
170
+
171
+ Already emitting OpenTelemetry? Point your OTLP exporter at `tj serve` — no SDK needed:
172
+
173
+ ```bash
174
+ tj serve &
175
+ export OTEL_EXPORTER_OTLP_ENDPOINT=http://127.0.0.1:7391
176
+ # run your agent as usual
177
+ ```
178
+
179
+ | Framework | OTel support |
180
+ |---|---|
181
+ | **Claude Code** | Built-in — `tj onboard --claude-code` |
182
+ | **OpenClaw** | Built-in (`diagnostics-otel` plugin) — [setup guide](docs/openclaw.md) |
183
+ | LlamaIndex | `opentelemetry-instrumentation-llama-index` |
184
+ | OpenAI Agents SDK | Built-in |
185
+ | Google ADK | Built-in |
186
+ | Strands Agent SDK (AWS) | Built-in |
187
+ | Haystack | Built-in |
188
+ | Pydantic AI | Built-in |
189
+ | Semantic Kernel | Built-in |
190
+
191
+ ---
192
+
193
+ ## CLI
194
+
195
+ ```
196
+ tj status
197
+ ```
198
+
199
+ ```
200
+ ● my-email-agent completed (2m 14s)
201
+
202
+ Cost today: $0.0340 / $5.0000 limit
203
+ Tokens: 12.4k in / 3.8k out
204
+ Tool calls: 47
205
+ Active session: sess-a1b2c3
206
+
207
+ send_email called (sensitive action: critical)
208
+ ```
209
+
210
+ https://github.com/user-attachments/assets/b94d13f6-1432-40d4-b093-6958d74f0e65
211
+
212
+ ```bash
213
+ tj status # current state, cost, active alerts
214
+ ocw traces # full span history with waterfall view
215
+ ocw cost --since 7d # cost breakdown by agent, model, day
216
+ ocw alerts # everything that fired while you were away
217
+ ocw budget # view and set daily/session cost limits
218
+ ocw drift # behavioral drift Z-scores vs baseline
219
+ ocw tools # tool call history with error rates
220
+ tj serve # start the web UI + REST API
221
+ ```
222
+
223
+ ---
224
+
225
+ ## Web UI
226
+
227
+ `tj serve` starts a local dashboard at `http://127.0.0.1:7391/`.
228
+
229
+ https://github.com/user-attachments/assets/ff09caec-3487-4542-8628-d62b7d92591f
230
+
231
+ - **Status** — agent overview with cost, tokens, tool calls, and active alerts
232
+ - **Traces** — trace list with span waterfall visualization
233
+ - **Cost** — breakdown by agent, model, day, or tool
234
+ - **Alerts** — alert history with severity filtering
235
+ - **Budget** — view and edit daily/session cost limits per agent
236
+ - **Drift** — behavioral drift report with Z-score analysis
237
+
238
+ No signup, no cloud — runs entirely on your machine.
239
+
240
+ ---
241
+
242
+ ## ocw vs LangSmith vs Langfuse
243
+
244
+ LangSmith and Langfuse are excellent for tracing LLM API calls and running evals on chat outputs. `tj` solves a different problem: **autonomous agents running unsupervised with real-world consequences**.
245
+
246
+ | | `tj` | LangSmith | Langfuse | Datadog LLM Obs |
247
+ |---|---|---|---|---|
248
+ | Signup required | ❌ | ✅ | ✅ | ✅ |
249
+ | Data leaves your machine | ❌ | ✅ | cloud only | ✅ |
250
+ | Real-time sensitive action alerts | ✅ | ❌ | ❌ | ❌ |
251
+ | Behavioral drift detection | ✅ | ❌ | ❌ | ❌ |
252
+ | Local-first, no cloud required | ✅ | ❌ | self-host only | ❌ |
253
+ | OTel GenAI SemConv native | ✅ | partial | partial | partial |
254
+ | NemoClaw sandbox events | ✅ | ❌ | ❌ | ❌ |
255
+ | Works with any agent/framework | ✅ | LangChain-first | partial | ❌ |
256
+ | Free, MIT licensed | ✅ | freemium | freemium | paid |
257
+
258
+ ---
259
+
260
+ ## Claude Code + coding agents
261
+
262
+ ### Claude Code
263
+
264
+ Monitor every Claude Code session — costs, tool calls, API requests, errors — with two commands:
265
+
266
+ ```bash
267
+ pip install "tokenjam[mcp]"
268
+ tj onboard --claude-code
269
+ # Restart Claude Code, then:
270
+ tj status --agent claude-code-<project>
271
+ ```
272
+
273
+ `tj onboard --claude-code` does everything in one shot:
274
+ - Creates a shared config at `~/.config/tj/config.toml` (one config for all projects)
275
+ - Writes OTLP exporter vars to `~/.claude/settings.json`
276
+ - Tags this project by writing `OTEL_RESOURCE_ATTRIBUTES` to `.claude/settings.json`
277
+ - Registers the MCP server globally (`claude mcp add --scope user tj -- tj mcp`)
278
+ - Installs a background daemon (launchd on macOS, systemd on Linux)
279
+ - Adds Docker harness-compatible OTLP env vars to `~/.zshrc`
280
+
281
+ **Claude Code must be restarted** after running `tj onboard --claude-code`.
282
+
283
+ **Adding more projects** — run once per project directory:
284
+
285
+ ```bash
286
+ cd /path/to/other-project
287
+ tj onboard --claude-code # tags this project, no reinstall needed
288
+ # Restart Claude Code
289
+ ```
290
+
291
+ Each project gets its own agent ID (`claude-code-<repo-name>`), all sharing one server and one ingest secret.
292
+
293
+ ### MCP server
294
+
295
+ The MCP server gives Claude Code direct access to your observability data inside the session. 13 tools available after restart:
296
+
297
+ | Tool | What it does |
298
+ |---|---|
299
+ | `get_status` | Current agent state — tokens, cost, active alerts |
300
+ | `get_budget_headroom` | Budget limit vs spend |
301
+ | `list_active_sessions` | All running sessions across agents |
302
+ | `list_agents` | All known agents with lifetime cost |
303
+ | `get_cost_summary` | Cost breakdown by day / agent / model |
304
+ | `list_alerts` | Alert history with severity filtering |
305
+ | `list_traces` | Recent traces with cost and duration |
306
+ | `get_trace` | Full span waterfall for a trace |
307
+ | `get_tool_stats` | Tool call counts and average duration |
308
+ | `get_drift_report` | Drift baseline vs latest session |
309
+ | `acknowledge_alert` | Mark an alert as acknowledged |
310
+ | `setup_project` | Configure a project for OCW telemetry |
311
+ | `open_dashboard` | Open the web UI (starts `tj serve` if needed) |
312
+
313
+ The MCP server opens DuckDB read-only — no lock conflicts with `tj serve`.
314
+
315
+ **Per-project tagging** — after installing globally, ask Claude Code:
316
+
317
+ > "Set up OCW for this project"
318
+
319
+ Claude calls `setup_project`, which writes `.claude/settings.json` with the right `OTEL_RESOURCE_ATTRIBUTES` for this project.
320
+
321
+ ### Codex
322
+
323
+ Monitor every Codex session — run once, globally:
324
+
325
+ ```bash
326
+ pip install "tokenjam[mcp]"
327
+ tj onboard --codex
328
+ ```
329
+
330
+ `tj onboard --codex` is project-agnostic. It writes to `~/.codex/config.toml` (Codex's single global config), so you only run it once — not once per project. Codex hardcodes `service.name="codex_exec"` in its binary, so all sessions appear under the same agent ID regardless of which repo you're working in.
331
+
332
+ `tj onboard --codex`:
333
+ - Writes an `[otel]` block and `[mcp_servers.ocw]` to `~/.codex/config.toml`
334
+ - Registers the MCP server so Codex can call OCW tools directly
335
+ - Installs the background daemon (launchd / systemd)
336
+
337
+ **Codex must be restarted** after running `tj onboard --codex`.
338
+
339
+ ```bash
340
+ tj status --agent codex_exec # check it's working
341
+ ```
342
+
343
+ The same 13 MCP tools available to Claude Code are available to Codex after restart.
344
+
345
+ ### Uninstalling
346
+
347
+ ```bash
348
+ # Remove all OCW data, config, daemon, MCP registration, and env vars:
349
+ ocw uninstall --yes
350
+
351
+ # Then remove the package:
352
+ pip uninstall tokenjam -y
353
+ ```
354
+
355
+ `tj uninstall` cleans up everything set by `tj onboard --claude-code`: daemon, MCP server, `~/.ocw/`, `~/.config/ocw/`, OTLP env vars in `~/.claude/settings.json`, `OTEL_RESOURCE_ATTRIBUTES` in every onboarded project's `.claude/settings.json`, and the harness env block in `~/.zshrc`.
356
+
357
+ ---
358
+
359
+ ## Supported frameworks
360
+
361
+ ### Python — provider patches
362
+
363
+ Intercept at the API level. Framework-agnostic.
364
+
365
+ ```python
366
+ from tokenjam.sdk.integrations.anthropic import patch_anthropic # Anthropic
367
+ from tokenjam.sdk.integrations.openai import patch_openai # OpenAI
368
+ from tokenjam.sdk.integrations.gemini import patch_gemini # Google Gemini
369
+ from tokenjam.sdk.integrations.bedrock import patch_bedrock # AWS Bedrock
370
+ from tokenjam.sdk.integrations.litellm import patch_litellm # LiteLLM (100+ providers)
371
+ ```
372
+
373
+ `patch_litellm()` covers all providers LiteLLM routes to (OpenAI, Anthropic, Bedrock, Vertex, Cohere, Mistral, Ollama, etc.). If you use LiteLLM, you don't need individual patches.
374
+
375
+ OpenAI-compatible providers (Groq, Together, Fireworks, xAI, Azure OpenAI) work via `patch_openai(base_url=...)`.
376
+
377
+ ### Python — framework patches
378
+
379
+ Instrument the framework's own abstractions:
380
+
381
+ ```python
382
+ from tokenjam.sdk.integrations.langchain import patch_langchain # BaseLLM + BaseTool
383
+ from tokenjam.sdk.integrations.langgraph import patch_langgraph # CompiledGraph
384
+ from tokenjam.sdk.integrations.crewai import patch_crewai # Task + Agent
385
+ from tokenjam.sdk.integrations.autogen import patch_autogen # ConversableAgent
386
+ from tokenjam.sdk.integrations.llamaindex import patch_llamaindex # Native OTel
387
+ from tokenjam.sdk.integrations.openai_agents_sdk import patch_openai_agents # Native OTel
388
+ from tokenjam.sdk.integrations.nemoclaw import watch_nemoclaw # NemoClaw Gateway
389
+ ```
390
+
391
+ Full framework support guide: [docs/framework-support.md](docs/framework-support.md)
392
+
393
+ ---
394
+
395
+ ## Alert channels
396
+
397
+ Configure where alerts go. Multiple channels work simultaneously.
398
+
399
+ ```toml
400
+ # .ocw/config.toml
401
+
402
+ [[alerts.channels]]
403
+ type = "ntfy"
404
+ topic = "my-agent-alerts" # push to your phone, free, no account
405
+
406
+ [[alerts.channels]]
407
+ type = "discord"
408
+ webhook_url = "https://discord.com/api/webhooks/..."
409
+
410
+ [[alerts.channels]]
411
+ type = "webhook"
412
+ url = "https://your-endpoint.com/alerts"
413
+ ```
414
+
415
+ Alert types: `sensitive_action` · `cost_budget_daily` · `cost_budget_session` · `session_duration` · `retry_loop` · `token_anomaly` · `schema_violation` · `drift_detected` · `failure_rate` · `network_egress_blocked` · `filesystem_access_denied` · `syscall_denied` · `inference_rerouted`
416
+
417
+ Full alert reference — trigger conditions, cooldown config, content stripping, all 6 channel types: [docs/alerts.md](docs/alerts.md)
418
+
419
+ ---
420
+
421
+ ## NemoClaw support
422
+
423
+ Running OpenClaw inside [NVIDIA NemoClaw](https://github.com/NVIDIA/NemoClaw)? `tj` connects to the OpenShell Gateway WebSocket and turns sandbox events — blocked network requests, filesystem denials, inference reroutes — into alerts.
424
+
425
+ ```python
426
+ from tokenjam.sdk.integrations.nemoclaw import watch_nemoclaw
427
+
428
+ observer = watch_nemoclaw()
429
+ asyncio.create_task(observer.connect())
430
+ ```
431
+
432
+ This is the observability layer that NemoClaw doesn't ship with.
433
+
434
+ Full event table and configuration: [docs/nemoclaw-integration.md](docs/nemoclaw-integration.md)
435
+
436
+ ---
437
+
438
+ ## Export and integrate
439
+
440
+ ```bash
441
+ ocw export --format otlp # forward to Grafana, Datadog, any OTel backend
442
+ ocw export --format openevals # openevals / agentevals trajectory evaluation
443
+ ocw export --format json # NDJSON
444
+ ocw export --format csv
445
+ ```
446
+
447
+ Prometheus metrics at `http://127.0.0.1:7391/metrics` when `tj serve` is running.
448
+
449
+ Export filtering flags, REST API endpoints, and API docs: [docs/export.md](docs/export.md)
450
+
451
+ ---
452
+
453
+ ## Architecture
454
+
455
+ ```mermaid
456
+ flowchart TD
457
+ Agent["Your agent"]
458
+
459
+ Agent --> Terminal["Coding agents\nClaude Code · Codex"]
460
+ Agent --> PythonSDK["Python SDK\n@watch + patch_*"]
461
+ Agent --> TypeScriptSDK["TypeScript SDK\n@tokenjam/sdk"]
462
+
463
+ Terminal --> OTLP["OTLP export"]
464
+ PythonSDK --> Exporter["TjSpanExporter"]
465
+ TypeScriptSDK --> HTTP["POST /api/v1/spans"]
466
+ OTLP --> HTTP
467
+
468
+ Exporter --> Ingest
469
+ HTTP --> Ingest
470
+
471
+ Ingest["IngestPipeline\nSanitize · Session continuity · Extract"]
472
+
473
+ Ingest --> Cost["CostEngine\npricing.toml"]
474
+ Ingest --> Alerts["AlertEngine\n13 types · 6 channels"]
475
+ Ingest --> Schema["SchemaValidator\nJSON Schema + infer"]
476
+
477
+ Cost --> DB["DuckDB\nlocal · embedded"]
478
+ Alerts --> DB
479
+ Schema --> DB
480
+
481
+ DB --> CLI["ocw CLI"]
482
+ DB --> API["REST API + Web UI\n:7391"]
483
+ DB --> MCP["MCP Server\n13 tools"]
484
+ DB --> Prom["Prometheus\n:7391/metrics"]
485
+ ```
486
+
487
+ Full architecture deep-dive — design principles, SDK internals, alert system, testing: [docs/architecture.md](docs/architecture.md)
488
+
489
+ ---
490
+
491
+ ## Configuration
492
+
493
+ ```toml
494
+ # .ocw/config.toml — generated by tj onboard
495
+
496
+ [defaults.budget]
497
+ daily_usd = 10.00
498
+
499
+ [agents.my-email-agent]
500
+ description = "Personal email management agent"
501
+
502
+ [agents.my-email-agent.budget]
503
+ daily_usd = 5.00
504
+ session_usd = 1.00
505
+
506
+ [[agents.my-email-agent.sensitive_actions]]
507
+ name = "send_email"
508
+ severity = "critical"
509
+
510
+ [agents.my-email-agent.drift]
511
+ enabled = true
512
+ baseline_sessions = 10
513
+ token_threshold = 2.0
514
+
515
+ [capture]
516
+ prompts = false
517
+ completions = false
518
+ tool_outputs = false
519
+
520
+ [storage]
521
+ path = "~/.ocw/telemetry.duckdb"
522
+ retention_days = 90
523
+ ```
524
+
525
+ Budget limits merge per-field: each agent inherits defaults unless overridden. Set via CLI (`tj budget --daily 10`), API, or web UI. Run `tj doctor` to verify.
526
+
527
+ Config file discovery order, full config schema, API auth, capture settings: [docs/configuration.md](docs/configuration.md)
528
+
529
+ ---
530
+
531
+ ## CLI reference
532
+
533
+ 16 commands: `onboard`, `doctor`, `status`, `traces`, `cost`, `alerts`, `budget`, `drift`, `tools`, `demo`, `export`, `mcp`, `serve`, `stop`, `uninstall`. All support `--json` for machine-readable output.
534
+
535
+ Global flags, per-command options, exit codes: [docs/cli-reference.md](docs/cli-reference.md)
536
+
537
+ ---
538
+
539
+ ## Examples
540
+
541
+ The [`examples/`](examples/) directory has runnable agents for every integration:
542
+
543
+ - **Single provider** — Anthropic, OpenAI, Gemini, Bedrock, OpenAI Agents SDK
544
+ - **Single framework** — LangChain, LangGraph, CrewAI, AutoGen, LlamaIndex
545
+ - **Multi-integration** — provider router, CrewAI + LangChain, RAG with fallback
546
+ - **Alerts and drift** — sensitive action alerts, budget breach, drift detection (no API keys needed)
547
+
548
+ ```bash
549
+ python examples/single_provider/anthropic_agent.py
550
+ python examples/alerts_and_drift/drift_demo.py # no API key needed
551
+ ```
552
+
553
+ See [`examples/README.md`](examples/README.md) for the full list.
554
+
555
+ ---
556
+
557
+ ## Agent Incident Library
558
+
559
+ Reproducible AI agent failures you can run in 30 seconds. No API keys, no config, no setup.
560
+
561
+ ```bash
562
+ ocw demo # list all scenarios
563
+ ocw demo retry-loop # run one
564
+ ocw demo retry-loop --json # machine-readable output
565
+ ```
566
+
567
+ | Scenario | What goes wrong | What OCW catches |
568
+ |---|---|---|
569
+ | [`retry-loop`](incidents/retry-loop/README.md) | Agent retries a failing tool in a loop, burning time and tokens | `retry_loop` + `failure_rate` alerts fire automatically |
570
+ | [`surprise-cost`](incidents/surprise-cost/README.md) | Model silently escalates from Haiku to Opus mid-chain | Per-model cost breakdown shows the $3+ you didn't expect |
571
+ | [`hallucination-drift`](incidents/hallucination-drift/README.md) | Agent behavior shifts — different tokens, different tools | `drift_detected` alert fires with Z-scores at session end |
572
+
573
+ Each scenario runs against an in-memory backend and produces a side-by-side comparison: what `print()` shows vs. what OCW reveals.
574
+
575
+ ---
576
+
577
+ ## Architecture
578
+
579
+ See [AGENTS.md](AGENTS.md) for codebase conventions.
580
+
581
+ PRs welcome. If you're adding a framework integration, open an issue first.
582
+
583
+ ---
584
+
585
+ ## Roadmap
586
+
587
+ **Shipped:**
588
+
589
+ - [x] `tj serve` background daemon (launchd / systemd)
590
+ - [x] Web UI with auto-polling (status, traces, cost, alerts, budget, drift)
591
+ - [x] LiteLLM provider patch (100+ providers)
592
+ - [x] `tj stop` and `tj uninstall`
593
+ - [x] Claude Code integration (`tj onboard --claude-code`)
594
+ - [x] Codex integration (`tj onboard --codex`)
595
+ - [x] OpenClaw integration (zero-code via `diagnostics-otel` plugin)
596
+ - [x] NemoClaw sandbox observer (WebSocket gateway events)
597
+ - [x] OTLP log-to-span pipeline (Claude Code log events)
598
+ - [x] `tj budget` CLI, API, and web UI
599
+ - [x] `tj drift` with Z-score reporting
600
+ - [x] Full pipeline wiring (alerts, schema, drift in `tj serve`)
601
+ - [x] MCP server — 13 tools for Claude Code
602
+
603
+ **Up next:**
604
+
605
+ - [ ] `tj watch` — live tail mode for spans
606
+ - [ ] `tj replay` — replay captured sessions against new model versions
607
+ - [ ] TypeScript framework patches (LangChain JS, OpenAI Agents SDK)
608
+ - [ ] Vercel AI SDK integration (TypeScript)
609
+ - [ ] Mastra integration (TypeScript)
610
+ - [ ] Azure AI Agent Service integration
611
+ - [ ] Docker image
612
+ - [ ] GitHub Actions for CI drift/cost checks
613
+
614
+ ---
615
+
616
+ <div align="center">
617
+
618
+ **[opencla.watch](https://opencla.watch)** · [PyPI](https://pypi.org/project/tokenjam/) · [npm](https://www.npmjs.com/package/@tokenjam/sdk)
619
+
620
+ MIT License · Built by [Metabuilder Labs](https://github.com/Metabuilder-Labs)
621
+
622
+ </div>