frugal-sdk-python 0.1.0__tar.gz → 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (78) hide show
  1. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/PKG-INFO +96 -1
  2. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/README.md +95 -0
  3. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/docs/metrics-reference.md +2 -1
  4. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/docs/quickstart.md +99 -0
  5. frugal_sdk_python-0.2.0/src/frugal_metrics/__init__.py +27 -0
  6. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/config.py +11 -0
  7. frugal_sdk_python-0.2.0/src/frugal_metrics/debug.py +109 -0
  8. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/instrument.py +8 -0
  9. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/otel.py +85 -5
  10. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/safe.py +11 -0
  11. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/semconv.py +3 -0
  12. frugal_sdk_python-0.2.0/src/frugal_metrics/shutdown.py +140 -0
  13. frugal_sdk_python-0.2.0/src/frugal_metrics/track.py +341 -0
  14. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/wrappers/anthropic.py +8 -0
  15. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/wrappers/bedrock.py +8 -0
  16. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/wrappers/openai.py +8 -0
  17. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/conftest.py +2 -0
  18. frugal_sdk_python-0.2.0/tests/test_debug.py +133 -0
  19. frugal_sdk_python-0.2.0/tests/test_shutdown.py +130 -0
  20. frugal_sdk_python-0.2.0/tests/test_track_ai_call.py +240 -0
  21. frugal_sdk_python-0.1.0/src/frugal_metrics/__init__.py +0 -13
  22. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/.github/workflows/cd.yml +0 -0
  23. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/.gitignore +0 -0
  24. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/LICENSES/Apache-2.0.txt +0 -0
  25. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/REUSE.toml +0 -0
  26. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/README.md +0 -0
  27. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/collector/otel-collector-config.yaml +0 -0
  28. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/docker-compose.yml +0 -0
  29. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/grafana/dashboards/frugal-metrics.json +0 -0
  30. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/grafana/provisioning/dashboards/frugal.yml +0 -0
  31. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/grafana/provisioning/datasources/prometheus.yml +0 -0
  32. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/mockllm/Dockerfile +0 -0
  33. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/mockllm/package.json +0 -0
  34. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/mockllm/src/server.js +0 -0
  35. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/prometheus/consoles/index.html.example +0 -0
  36. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/prometheus/consoles/models.html +0 -0
  37. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/prometheus/consoles/overview.html +0 -0
  38. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/prometheus/consoles/sites.html +0 -0
  39. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/prometheus/prometheus.yml +0 -0
  40. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/Dockerfile +0 -0
  41. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/__init__.py +0 -0
  42. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/clients.py +0 -0
  43. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/__init__.py +0 -0
  44. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/anon.py +0 -0
  45. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/classify.py +0 -0
  46. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/deferred.py +0 -0
  47. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/fail.py +0 -0
  48. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/pipeline.py +0 -0
  49. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/summarize.py +0 -0
  50. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/translate.py +0 -0
  51. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/via_blocked_lib.py +0 -0
  52. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/handlers/via_unblocked_lib.py +0 -0
  53. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/internal_lib/__init__.py +0 -0
  54. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/internal_lib/ai_helper.py +0 -0
  55. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/main.py +0 -0
  56. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/sibling_lib/__init__.py +0 -0
  57. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/app/sibling_lib/ai_helper.py +0 -0
  58. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/pyproject.toml +0 -0
  59. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/sample-app/requirements.txt +0 -0
  60. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/examples/test-env/scripts/verify.sh +0 -0
  61. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/pyproject.toml +0 -0
  62. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/block_hash_lru.py +0 -0
  63. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/caller.py +0 -0
  64. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/prompt_analyzer.py +0 -0
  65. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/streams.py +0 -0
  66. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/wrappers/__init__.py +0 -0
  67. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/src/frugal_metrics/wrappers/bedrock_models.py +0 -0
  68. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/__init__.py +0 -0
  69. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_bedrock_models.py +0 -0
  70. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_block_hash_lru.py +0 -0
  71. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_caller.py +0 -0
  72. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_caller_edge_cases.py +0 -0
  73. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_config.py +0 -0
  74. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_prompt_analyzer.py +0 -0
  75. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_wrap_anthropic.py +0 -0
  76. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_wrap_bedrock.py +0 -0
  77. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_wrap_openai.py +0 -0
  78. {frugal_sdk_python-0.1.0 → frugal_sdk_python-0.2.0}/tests/test_wrap_openai_analyzer.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.4
2
2
  Name: frugal-sdk-python
3
- Version: 0.1.0
3
+ Version: 0.2.0
4
4
  Summary: Instrumentation for Cost-to-Code attribution.
5
5
  Project-URL: Homepage, https://frugal.co
6
6
  Author: Frugal AI Inc.
@@ -140,6 +140,50 @@ profile prefixes (`us.`, `eu.`, `jp.`, `apac.`, `au.`, `global.`) are
140
140
  preserved in `gen_ai.request.model`. Async clients (`aiobotocore` /
141
141
  `aioboto3`) are detected automatically. See [`docs/quickstart.md`](docs/quickstart.md#native-aws-bedrock-wrap_bedrock) for the full Bedrock walkthrough.
142
142
 
143
+ ## Custom call sites
144
+
145
+ For AI calls that don't go through a wrapped client — an HTTP call to a model gateway, a CLI shell-out, or an internal helper that returns raw usage — mark the call site yourself and feed in the usage you have. You get the same metrics and the same caller attribution as the auto-wrappers.
146
+
147
+ ```python
148
+ from frugal_metrics import track_ai_call
149
+
150
+ with track_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514") as span:
151
+ res = my_custom_llm_call(prompt)
152
+ span.set_usage(
153
+ input_tokens=res.usage.input_tokens,
154
+ output_tokens=res.usage.output_tokens,
155
+ response_model=res.model,
156
+ )
157
+ ```
158
+
159
+ The `with` block times the call and emits metrics on exit — or an error metric if the block raises (the exception always propagates). Works the same around an `await` in async code. Call `span.set_usage(...)` once the response is back; if you never call it, you still get a call-count and latency point.
160
+
161
+ For calls that span function boundaries — where a single `with` block won't fit — use the manual handle:
162
+
163
+ ```python
164
+ from frugal_metrics import start_ai_call
165
+
166
+ span = start_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514")
167
+ try:
168
+ res = my_custom_llm_call(prompt)
169
+ span.success(
170
+ input_tokens=res.usage.input_tokens,
171
+ output_tokens=res.usage.output_tokens,
172
+ response_model=res.model,
173
+ )
174
+ except BaseException as exc:
175
+ span.error(exc)
176
+ raise
177
+ ```
178
+
179
+ `success()` and `error()` finalize the call exactly once and are idempotent — extra calls are ignored.
180
+
181
+ **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
182
+
183
+ **Usage** — `set_usage(...)` / `success(...)` kwargs (all optional): `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `response_model`.
184
+
185
+ The same env-var configuration, safety guarantees, and exported metrics apply. See [`docs/quickstart.md`](docs/quickstart.md#custom-call-sites) for the full custom call-site walkthrough.
186
+
143
187
  ## Streaming
144
188
 
145
189
  Streaming works with no extra setup. The wrapper returns a drop-in replacement that passes events through and emits metrics when the stream finishes.
@@ -180,6 +224,57 @@ FRUGAL_PATH_BLOCK_LIST="src/acme_ai_helpers,libs/llm_utils"
180
224
  - If any part of our instrumentation fails at runtime (OTel init, stack walking, metric emission), the original SDK call executes normally. We never raise our own exceptions into your code.
181
225
  - Wrapping is idempotent — calling `wrap_openai(client)` twice on the same client is safe.
182
226
 
227
+ ## Debugging
228
+
229
+ The SDK is silent by design — when misconfigured it degrades to a no-op. To see *why* it is doing nothing (missing env var, instrument-build failure, cardinality-cap hit, a metric export that keeps failing), register a debug callback:
230
+
231
+ ```python
232
+ from frugal_metrics import set_debug_callback
233
+
234
+ def on_debug(level, message, context=None):
235
+ # level: "info" | "warn" | "error"
236
+ # message: human-readable string
237
+ # context: optional dict of structured detail, e.g. {"stage": "setup_failed"}
238
+ getattr(my_logger, level if level != "warn" else "warning")(
239
+ "[frugal-metrics] %s", message, extra=context or {}
240
+ )
241
+
242
+ set_debug_callback(on_debug)
243
+ ```
244
+
245
+ This is a **structural-diagnostics hook, not a per-call trace**. It only fires for structural or persistent problems — missing/invalid configuration, setup failures, cardinality-cap hits, and *consecutive* export failures (one de-duped `error`, re-armed on the next success). Ordinary transient per-call API errors are never emitted — those are your own API errors and already visible to you.
246
+
247
+ Notes:
248
+
249
+ - **A throwing callback is swallowed.** If your callback raises, the exception never escapes into the SDK or your code.
250
+ - **Issues are replayed.** Configuration runs lazily and may happen before you register, so init/config diagnostics emitted beforehand are held in a small bounded buffer and replayed in order the moment you call `set_debug_callback`. Register once at startup to catch them all.
251
+ - The callback is global and thread-safe. Pass `None` to clear it.
252
+
253
+ ## Flushing & graceful shutdown
254
+
255
+ Metrics export on a periodic interval (default 60s). A short-lived process — a CLI, a job, a serverless invocation, a test run — can exit before the next export, silently dropping its final window of metrics. Use `flush` / `shutdown` to avoid that:
256
+
257
+ ```python
258
+ from frugal_metrics import flush, shutdown
259
+
260
+ flush() # force an immediate export; SDK keeps recording afterwards
261
+ shutdown() # flush + tear down the provider (rebuilt lazily on next wrap)
262
+ ```
263
+
264
+ Both are **safe no-ops when the SDK was never initialized** and never raise.
265
+
266
+ For processes that exit on their own, opt in to automatic flush-on-exit:
267
+
268
+ ```python
269
+ from frugal_metrics import enable_auto_shutdown
270
+
271
+ enable_auto_shutdown() # flush + shutdown via atexit, plus SIGTERM / SIGINT
272
+ ```
273
+
274
+ `enable_auto_shutdown` always registers an `atexit` hook and, when `handle_signals=True` (the default), installs SIGTERM/SIGINT handlers that flush+shutdown and then **chain to whatever handler was previously installed** (so your own handler still runs). Off the main thread, signal registration is unavailable and it falls back to atexit-only. It is **idempotent** — repeat calls register nothing extra.
275
+
276
+ It is **opt-in** because installing signal handlers is intrusive. If you run your own signal handling, do **not** call `enable_auto_shutdown(handle_signals=True)`; prefer calling `shutdown()` from inside your own handler instead.
277
+
183
278
  ## Verifying the setup
184
279
 
185
280
  Set the OTLP endpoint to empty to print metrics to the console instead of exporting them:
@@ -111,6 +111,50 @@ profile prefixes (`us.`, `eu.`, `jp.`, `apac.`, `au.`, `global.`) are
111
111
  preserved in `gen_ai.request.model`. Async clients (`aiobotocore` /
112
112
  `aioboto3`) are detected automatically. See [`docs/quickstart.md`](docs/quickstart.md#native-aws-bedrock-wrap_bedrock) for the full Bedrock walkthrough.
113
113
 
114
+ ## Custom call sites
115
+
116
+ For AI calls that don't go through a wrapped client — an HTTP call to a model gateway, a CLI shell-out, or an internal helper that returns raw usage — mark the call site yourself and feed in the usage you have. You get the same metrics and the same caller attribution as the auto-wrappers.
117
+
118
+ ```python
119
+ from frugal_metrics import track_ai_call
120
+
121
+ with track_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514") as span:
122
+ res = my_custom_llm_call(prompt)
123
+ span.set_usage(
124
+ input_tokens=res.usage.input_tokens,
125
+ output_tokens=res.usage.output_tokens,
126
+ response_model=res.model,
127
+ )
128
+ ```
129
+
130
+ The `with` block times the call and emits metrics on exit — or an error metric if the block raises (the exception always propagates). Works the same around an `await` in async code. Call `span.set_usage(...)` once the response is back; if you never call it, you still get a call-count and latency point.
131
+
132
+ For calls that span function boundaries — where a single `with` block won't fit — use the manual handle:
133
+
134
+ ```python
135
+ from frugal_metrics import start_ai_call
136
+
137
+ span = start_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514")
138
+ try:
139
+ res = my_custom_llm_call(prompt)
140
+ span.success(
141
+ input_tokens=res.usage.input_tokens,
142
+ output_tokens=res.usage.output_tokens,
143
+ response_model=res.model,
144
+ )
145
+ except BaseException as exc:
146
+ span.error(exc)
147
+ raise
148
+ ```
149
+
150
+ `success()` and `error()` finalize the call exactly once and are idempotent — extra calls are ignored.
151
+
152
+ **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
153
+
154
+ **Usage** — `set_usage(...)` / `success(...)` kwargs (all optional): `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `response_model`.
155
+
156
+ The same env-var configuration, safety guarantees, and exported metrics apply. See [`docs/quickstart.md`](docs/quickstart.md#custom-call-sites) for the full custom call-site walkthrough.
157
+
114
158
  ## Streaming
115
159
 
116
160
  Streaming works with no extra setup. The wrapper returns a drop-in replacement that passes events through and emits metrics when the stream finishes.
@@ -151,6 +195,57 @@ FRUGAL_PATH_BLOCK_LIST="src/acme_ai_helpers,libs/llm_utils"
151
195
  - If any part of our instrumentation fails at runtime (OTel init, stack walking, metric emission), the original SDK call executes normally. We never raise our own exceptions into your code.
152
196
  - Wrapping is idempotent — calling `wrap_openai(client)` twice on the same client is safe.
153
197
 
198
+ ## Debugging
199
+
200
+ The SDK is silent by design — when misconfigured it degrades to a no-op. To see *why* it is doing nothing (missing env var, instrument-build failure, cardinality-cap hit, a metric export that keeps failing), register a debug callback:
201
+
202
+ ```python
203
+ from frugal_metrics import set_debug_callback
204
+
205
+ def on_debug(level, message, context=None):
206
+ # level: "info" | "warn" | "error"
207
+ # message: human-readable string
208
+ # context: optional dict of structured detail, e.g. {"stage": "setup_failed"}
209
+ getattr(my_logger, level if level != "warn" else "warning")(
210
+ "[frugal-metrics] %s", message, extra=context or {}
211
+ )
212
+
213
+ set_debug_callback(on_debug)
214
+ ```
215
+
216
+ This is a **structural-diagnostics hook, not a per-call trace**. It only fires for structural or persistent problems — missing/invalid configuration, setup failures, cardinality-cap hits, and *consecutive* export failures (one de-duped `error`, re-armed on the next success). Ordinary transient per-call API errors are never emitted — those are your own API errors and already visible to you.
217
+
218
+ Notes:
219
+
220
+ - **A throwing callback is swallowed.** If your callback raises, the exception never escapes into the SDK or your code.
221
+ - **Issues are replayed.** Configuration runs lazily and may happen before you register, so init/config diagnostics emitted beforehand are held in a small bounded buffer and replayed in order the moment you call `set_debug_callback`. Register once at startup to catch them all.
222
+ - The callback is global and thread-safe. Pass `None` to clear it.
223
+
224
+ ## Flushing & graceful shutdown
225
+
226
+ Metrics export on a periodic interval (default 60s). A short-lived process — a CLI, a job, a serverless invocation, a test run — can exit before the next export, silently dropping its final window of metrics. Use `flush` / `shutdown` to avoid that:
227
+
228
+ ```python
229
+ from frugal_metrics import flush, shutdown
230
+
231
+ flush() # force an immediate export; SDK keeps recording afterwards
232
+ shutdown() # flush + tear down the provider (rebuilt lazily on next wrap)
233
+ ```
234
+
235
+ Both are **safe no-ops when the SDK was never initialized** and never raise.
236
+
237
+ For processes that exit on their own, opt in to automatic flush-on-exit:
238
+
239
+ ```python
240
+ from frugal_metrics import enable_auto_shutdown
241
+
242
+ enable_auto_shutdown() # flush + shutdown via atexit, plus SIGTERM / SIGINT
243
+ ```
244
+
245
+ `enable_auto_shutdown` always registers an `atexit` hook and, when `handle_signals=True` (the default), installs SIGTERM/SIGINT handlers that flush+shutdown and then **chain to whatever handler was previously installed** (so your own handler still runs). Off the main thread, signal registration is unavailable and it falls back to atexit-only. It is **idempotent** — repeat calls register nothing extra.
246
+
247
+ It is **opt-in** because installing signal handlers is intrusive. If you run your own signal handling, do **not** call `enable_auto_shutdown(handle_signals=True)`; prefer calling `shutdown()` from inside your own handler instead.
248
+
154
249
  ## Verifying the setup
155
250
 
156
251
  Set the OTLP endpoint to empty to print metrics to the console instead of exporting them:
@@ -45,7 +45,7 @@ calls, token usage) are 100% — never sampled.
45
45
 
46
46
  | Attribute | Example |
47
47
  | -------------------------- | ------------------------------------------ |
48
- | `gen_ai.system` | `openai` \| `anthropic` \| `aws.bedrock` \| `vertex_ai` |
48
+ | `gen_ai.system` | `openai` \| `anthropic` \| `aws.bedrock` \| `vertex_ai` \| `custom` |
49
49
  | `gen_ai.operation.name` | `chat` \| `text_completion` \| `embeddings` |
50
50
  | `gen_ai.request.model` | `gpt-4o` / `claude-3-5-sonnet-latest` / … |
51
51
  | `gen_ai.response.model` | Provider-returned model id (often identical)|
@@ -110,6 +110,7 @@ attribute describing where our own code failed. Non-exhaustive list:
110
110
  | Claude on AWS Bedrock (via Anthropic SDK) | `wrap_anthropic(AnthropicBedrock())` | `aws.bedrock` |
111
111
  | AWS Bedrock — any model, AWS SDKs directly | `wrap_bedrock` | `aws.bedrock` |
112
112
  | Claude on Google Vertex | `wrap_anthropic(AnthropicVertex())` | `vertex_ai` |
113
+ | Custom / manual call sites | `track_ai_call` / `start_ai_call` | `custom` (override via `system=`) |
113
114
 
114
115
  `wrap_bedrock` covers all four `bedrock-runtime` operations (`InvokeModel`,
115
116
  `InvokeModelWithResponseStream`, `Converse`, `ConverseStream`) on every model
@@ -198,6 +198,65 @@ client = wrap_anthropic(AnthropicVertex(region="us-central1", project_id="my-pro
198
198
  > integrations are not yet instrumented. The block-list approach below
199
199
  > usually covers framework wrappers in the meantime.
200
200
 
201
+ ## Custom call sites
202
+
203
+ For AI calls that don't go through a wrapped client — an HTTP call to a model
204
+ gateway, a CLI shell-out, or an internal helper that returns raw usage — mark
205
+ the call site yourself and feed in the usage you have. You get the same metrics
206
+ (tagged `gen_ai.system = custom` by default) and the same caller attribution as
207
+ the auto-wrappers.
208
+
209
+ ```python
210
+ from frugal_metrics import track_ai_call
211
+
212
+ with track_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514") as span:
213
+ res = my_custom_llm_call(prompt)
214
+ span.set_usage(
215
+ input_tokens=res.usage.input_tokens,
216
+ output_tokens=res.usage.output_tokens,
217
+ response_model=res.model,
218
+ )
219
+ ```
220
+
221
+ The `with` block times the call and emits metrics on exit — or an error metric
222
+ if the block raises (the exception always propagates). Works the same around an
223
+ `await` in async code. Call `span.set_usage(...)` once the response is back; if
224
+ you never call it, you still get a call-count and latency point.
225
+
226
+ For calls that span function boundaries — where a single `with` block won't fit
227
+ — use the manual handle:
228
+
229
+ ```python
230
+ from frugal_metrics import start_ai_call
231
+
232
+ span = start_ai_call(system="anthropic", request_model="claude-sonnet-4-20250514")
233
+ try:
234
+ res = my_custom_llm_call(prompt)
235
+ span.success(
236
+ input_tokens=res.usage.input_tokens,
237
+ output_tokens=res.usage.output_tokens,
238
+ response_model=res.model,
239
+ )
240
+ except BaseException as exc:
241
+ span.error(exc)
242
+ raise
243
+ ```
244
+
245
+ `success()` and `error()` finalize the call exactly once and are idempotent —
246
+ extra calls are ignored.
247
+
248
+ **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional):
249
+ `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default
250
+ `"chat"`), `request_model` (default `"<unknown>"`). To also run the sampled
251
+ prompt analyzer, pass any of `messages` / `system_prompt` / `tools` /
252
+ `response_format` / `max_tokens`.
253
+
254
+ **Usage** — `set_usage(...)` / `success(...)` kwargs (all optional):
255
+ `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`,
256
+ `reasoning_tokens`, `response_model`.
257
+
258
+ The same env-var configuration, safety guarantees, and exported metrics apply.
259
+
201
260
  ## Attributing the right caller
202
261
 
203
262
  By default we walk the stack and pick the first frame that is:
@@ -272,6 +331,46 @@ The token-side metrics (`gen_ai.client.token.usage`, `frugal.gen_ai.calls`,
272
331
  `frugal.gen_ai.cache_*`, `frugal.gen_ai.output_cap_utilization`) are 100% —
273
332
  never sampled.
274
333
 
334
+ ## See why nothing is exporting (debug callback)
335
+
336
+ The SDK is silent by design, so when it no-ops you get no feedback. Register a
337
+ debug callback once at startup to surface structural diagnostics (missing config,
338
+ setup failures, cardinality-cap hits, persistent export failures) through your own
339
+ logger:
340
+
341
+ ```python
342
+ from frugal_metrics import set_debug_callback
343
+
344
+ def on_debug(level, message, context=None):
345
+ print(f"[frugal-metrics] {level}: {message}", context or {})
346
+
347
+ set_debug_callback(on_debug)
348
+ ```
349
+
350
+ It is a structural-diagnostics hook, not a per-call trace. A throwing callback is
351
+ swallowed, and issues emitted before you register are replayed on registration —
352
+ so register early. Pass `None` to clear it.
353
+
354
+ ## Flush on shutdown
355
+
356
+ Export runs on an interval, so a short-lived process can exit before the final
357
+ batch ships. Flush (or shut down) before exit:
358
+
359
+ ```python
360
+ from frugal_metrics import flush, shutdown, enable_auto_shutdown
361
+
362
+ flush() # immediate export, keep recording
363
+ shutdown() # flush + tear down (safe even if never initialized)
364
+
365
+ enable_auto_shutdown() # or: opt in to auto flush+shutdown on exit/SIGTERM/SIGINT
366
+ ```
367
+
368
+ `flush` / `shutdown` are safe no-ops when the SDK was never initialized.
369
+ `enable_auto_shutdown` registers an `atexit` hook and (by default) chained
370
+ SIGTERM/SIGINT handlers, and is idempotent; if you manage your own signal
371
+ handlers, call it with `handle_signals=False` (or just call `shutdown()` from
372
+ your handler).
373
+
275
374
  ## Verifying the setup
276
375
 
277
376
  Point the library at a local collector (or set the endpoint empty to print
@@ -0,0 +1,27 @@
1
+ """frugal-metrics: call-site instrumentation for AI API calls."""
2
+
3
+ from importlib.metadata import PackageNotFoundError, version
4
+
5
+ from frugal_metrics.debug import set_debug_callback
6
+ from frugal_metrics.shutdown import enable_auto_shutdown, flush, shutdown
7
+ from frugal_metrics.track import AICallSpan, start_ai_call, track_ai_call
8
+ from frugal_metrics.wrappers.openai import wrap_openai
9
+ from frugal_metrics.wrappers.anthropic import wrap_anthropic
10
+ from frugal_metrics.wrappers.bedrock import wrap_bedrock
11
+
12
+ __all__ = [
13
+ "wrap_openai",
14
+ "wrap_anthropic",
15
+ "wrap_bedrock",
16
+ "track_ai_call",
17
+ "start_ai_call",
18
+ "AICallSpan",
19
+ "set_debug_callback",
20
+ "flush",
21
+ "shutdown",
22
+ "enable_auto_shutdown",
23
+ ]
24
+ try:
25
+ __version__ = version("frugal-sdk-python")
26
+ except PackageNotFoundError:
27
+ __version__ = "0+unknown"
@@ -35,6 +35,8 @@ import os
35
35
  import threading
36
36
  from dataclasses import dataclass, field
37
37
 
38
+ from frugal_metrics.debug import emit_log
39
+
38
40
  logger = logging.getLogger("frugal_metrics")
39
41
 
40
42
  _ENV_API_KEY = "FRUGAL_API_KEY"
@@ -92,6 +94,7 @@ def _warn_once(message: str) -> None:
92
94
  return
93
95
  _warned_once = True
94
96
  logger.warning("frugal-metrics disabled: %s", message)
97
+ emit_log("warn", f"disabled: {message}")
95
98
 
96
99
 
97
100
  def load_config() -> Config | None:
@@ -207,6 +210,10 @@ def _parse_int(raw: str | None, *, default: int) -> int:
207
210
  raw,
208
211
  default,
209
212
  )
213
+ emit_log(
214
+ "warn",
215
+ f"could not parse int env var value {raw!r}; using default {default}",
216
+ )
210
217
  return default
211
218
 
212
219
 
@@ -221,6 +228,10 @@ def _parse_float(raw: str | None, *, default: float) -> float:
221
228
  raw,
222
229
  default,
223
230
  )
231
+ emit_log(
232
+ "warn",
233
+ f"could not parse float env var value {raw!r}; using default {default}",
234
+ )
224
235
  return default
225
236
 
226
237
 
@@ -0,0 +1,109 @@
1
+ """Debug / log callback.
2
+
3
+ Lets a customer observe the SDK's own structural diagnostics — missing/invalid
4
+ configuration, instrument-build failures, cardinality-cap hits, persistent
5
+ export failures — via a single registered callback. This is NOT a per-call
6
+ trace: only structural/persistent problems are emitted, never ordinary transient
7
+ per-call API errors (those are the customer's own API errors).
8
+
9
+ Safe by construction: a throwing customer callback can never escape (every
10
+ invocation is wrapped in try/except), and emissions made before a callback is
11
+ registered are held in a small bounded buffer and replayed on registration.
12
+ """
13
+
14
+ from __future__ import annotations
15
+
16
+ import logging
17
+ import threading
18
+ from typing import Any, Callable
19
+
20
+ logger = logging.getLogger("frugal_metrics")
21
+
22
+ # level is one of "info" | "warn" | "error".
23
+ DebugCallback = Callable[[str, str, "dict[str, Any] | None"], None]
24
+
25
+ # Cap the replay buffer so an app that never registers a callback can't grow
26
+ # memory unbounded. Oldest entries are dropped when full.
27
+ _MAX_BUFFER = 50
28
+
29
+ _lock = threading.Lock()
30
+ _callback: DebugCallback | None = None
31
+ _buffer: list[tuple[str, str, "dict[str, Any] | None"]] = []
32
+ _once_keys: set[str] = set()
33
+
34
+
35
+ def _invoke(
36
+ cb: DebugCallback,
37
+ level: str,
38
+ message: str,
39
+ context: dict[str, Any] | None,
40
+ ) -> None:
41
+ try:
42
+ cb(level, message, context)
43
+ except Exception: # noqa: BLE001
44
+ # A throwing customer callback must never escape into SDK / customer code.
45
+ logger.debug("frugal-metrics: debug callback raised", exc_info=True)
46
+
47
+
48
+ def set_debug_callback(cb: DebugCallback | None) -> None:
49
+ """Register (or clear, with ``None``) the debug callback.
50
+
51
+ On registration of a non-None callback, the bounded replay buffer of
52
+ diagnostics emitted before registration is replayed through the callback in
53
+ order, then cleared.
54
+ """
55
+ global _callback
56
+ with _lock:
57
+ _callback = cb
58
+ if cb is None:
59
+ return
60
+ pending = _buffer[:]
61
+ _buffer.clear()
62
+ for level, message, context in pending:
63
+ _invoke(cb, level, message, context)
64
+
65
+
66
+ def emit_log(level: str, message: str, context: dict[str, Any] | None = None) -> None:
67
+ """Emit a structural diagnostic.
68
+
69
+ Invokes the registered callback (inside a try/except so a throwing callback
70
+ never escapes); if none is registered yet, the entry is held in the bounded
71
+ replay buffer.
72
+ """
73
+ with _lock:
74
+ cb = _callback
75
+ if cb is None:
76
+ _buffer.append((level, message, context))
77
+ if len(_buffer) > _MAX_BUFFER:
78
+ _buffer.pop(0)
79
+ return
80
+ _invoke(cb, level, message, context)
81
+
82
+
83
+ def emit_log_once(
84
+ key: str,
85
+ level: str,
86
+ message: str,
87
+ context: dict[str, Any] | None = None,
88
+ ) -> None:
89
+ """Emit a diagnostic at most once per ``key``.
90
+
91
+ The de-dupe set persists for the process lifetime — use this for steady-state
92
+ conditions that would otherwise spam (cardinality-cap hits, repeated setup
93
+ failures). For the consecutive export-failure signal, which needs to re-arm,
94
+ use the explicit reset in the export-health wrapper instead.
95
+ """
96
+ with _lock:
97
+ if key in _once_keys:
98
+ return
99
+ _once_keys.add(key)
100
+ emit_log(level, message, context)
101
+
102
+
103
+ def reset_debug_for_tests() -> None:
104
+ """Test-only hook — clear callback, replay buffer, and once-keys."""
105
+ global _callback
106
+ with _lock:
107
+ _callback = None
108
+ _buffer.clear()
109
+ _once_keys.clear()
@@ -21,6 +21,7 @@ from frugal_metrics.caller import (
21
21
  resolve_caller,
22
22
  )
23
23
  from frugal_metrics.config import Config
24
+ from frugal_metrics.debug import emit_log_once
24
25
  from frugal_metrics.otel import Instruments, get_instruments
25
26
  from frugal_metrics.prompt_analyzer import (
26
27
  RequestPayload,
@@ -84,6 +85,13 @@ def start_call(
84
85
  resolver = caller_resolver or default_caller_resolver(config, ctx)
85
86
  caller = resolver()
86
87
  if consume_cap_hit_event():
88
+ emit_log_once(
89
+ "cardinality_cap",
90
+ "warn",
91
+ "call-site cardinality cap hit; further distinct (file, function) call "
92
+ "sites are attributed to an overflow bucket. Raise FRUGAL_MAX_CALL_SITES "
93
+ "if this is unexpected.",
94
+ )
87
95
  try:
88
96
  cap_attrs: dict[str, Any] = {
89
97
  semconv.FRUGAL_PROJECT_ID: config.project_id,
@@ -19,6 +19,7 @@ from opentelemetry.sdk.metrics import MeterProvider
19
19
  from opentelemetry.sdk.metrics.export import (
20
20
  ConsoleMetricExporter,
21
21
  MetricExporter,
22
+ MetricExportResult,
22
23
  PeriodicExportingMetricReader,
23
24
  )
24
25
  from opentelemetry.sdk.metrics.view import (
@@ -28,6 +29,7 @@ from opentelemetry.sdk.metrics.view import (
28
29
  from opentelemetry.sdk.resources import Resource
29
30
 
30
31
  from frugal_metrics import semconv
32
+ from frugal_metrics.debug import emit_log
31
33
 
32
34
  if TYPE_CHECKING:
33
35
  from opentelemetry.metrics import Counter, Histogram, Meter
@@ -145,7 +147,11 @@ def get_instruments(config: "Config") -> Instruments:
145
147
  with _lock:
146
148
  if _instruments is not None:
147
149
  return _instruments
148
- _instruments = _build_instruments(config)
150
+ try:
151
+ _instruments = _build_instruments(config)
152
+ except Exception as exc: # noqa: BLE001
153
+ emit_log("error", f"failed to initialize metrics: {exc}")
154
+ raise
149
155
  return _instruments
150
156
 
151
157
 
@@ -285,15 +291,89 @@ def _build_exporter(config: "Config") -> MetricExporter:
285
291
  logger.info(
286
292
  "frugal-metrics: no OTLP endpoint configured; falling back to console exporter"
287
293
  )
288
- return ConsoleMetricExporter()
294
+ return wrap_exporter_for_health(ConsoleMetricExporter())
289
295
 
290
296
  headers = _parse_headers(config.otlp_headers)
291
- return OTLPMetricExporter(
292
- endpoint=config.otlp_endpoint,
293
- headers=headers or None,
297
+ return wrap_exporter_for_health(
298
+ OTLPMetricExporter(
299
+ endpoint=config.otlp_endpoint,
300
+ headers=headers or None,
301
+ )
294
302
  )
295
303
 
296
304
 
305
+ class _HealthCheckingExporter(MetricExporter):
306
+ """Wrap an exporter so persistent export failures surface as a single
307
+ de-duped ``error`` debug callback.
308
+
309
+ The periodic reader retries silently forever; without this, a wrong
310
+ endpoint / bad token / down backend is invisible to the customer. We emit on
311
+ the FIRST failure after a success (0->1 edge) and re-arm on the next success,
312
+ so a flapping backend produces one alert per failure run, not one per export
313
+ tick. Every other attribute delegates to the inner exporter.
314
+ """
315
+
316
+ def __init__(self, inner: MetricExporter) -> None:
317
+ # Do NOT call super().__init__ — we delegate everything to the inner
318
+ # exporter, including the preferred temporality/aggregation it was built
319
+ # with. Accessing those goes through __getattr__.
320
+ self._inner = inner
321
+ self._consecutive_failures = 0
322
+
323
+ def export( # type: ignore[override]
324
+ self,
325
+ metrics_data: Any,
326
+ timeout_millis: float = 10_000,
327
+ **kwargs: Any,
328
+ ) -> MetricExportResult:
329
+ result = self._inner.export(
330
+ metrics_data, timeout_millis=timeout_millis, **kwargs
331
+ )
332
+ if result == MetricExportResult.SUCCESS:
333
+ self._consecutive_failures = 0
334
+ else:
335
+ self._consecutive_failures += 1
336
+ if self._consecutive_failures == 1:
337
+ emit_log("error", "metric export failing")
338
+ return result
339
+
340
+ def force_flush(self, timeout_millis: float = 10_000) -> bool:
341
+ return self._inner.force_flush(timeout_millis=timeout_millis)
342
+
343
+ def shutdown(self, timeout_millis: float = 30_000, **kwargs: Any) -> None:
344
+ self._inner.shutdown(timeout_millis=timeout_millis, **kwargs)
345
+
346
+ def __getattr__(self, name: str) -> Any:
347
+ # Delegate preferred_temporality / preferred_aggregation and any other
348
+ # attribute the SDK reads off the exporter to the inner instance.
349
+ return getattr(self._inner, name)
350
+
351
+
352
+ def wrap_exporter_for_health(exporter: MetricExporter) -> MetricExporter:
353
+ """Wrap an exporter so persistent export failures emit one de-duped error."""
354
+ return _HealthCheckingExporter(exporter)
355
+
356
+
357
+ def flush_metrics() -> None:
358
+ """Force an immediate export of buffered metrics. No-op when uninitialized."""
359
+ if _instruments is None:
360
+ return
361
+ _instruments.meter_provider.force_flush()
362
+
363
+
364
+ def shutdown_metrics() -> None:
365
+ """Shut down the provider (flushes + stops readers) and drop the cache so a
366
+ later wrap call lazily rebuilds. No-op when uninitialized.
367
+ """
368
+ global _instruments
369
+ with _lock:
370
+ if _instruments is None:
371
+ return
372
+ provider = _instruments.meter_provider
373
+ _instruments = None
374
+ provider.shutdown()
375
+
376
+
297
377
  def _parse_headers(raw: str | None) -> dict[str, str]:
298
378
  """Parse OTEL-style header strings: key1=value1,key2=value2."""
299
379
  if not raw: