frugal-sdk-python 0.5.0__tar.gz → 0.7.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (93) hide show
  1. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/PKG-INFO +41 -15
  2. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/README.md +40 -14
  3. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/docs/metrics-reference.md +34 -12
  4. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/docs/quickstart.md +39 -18
  5. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/mockllm/package.json +1 -1
  6. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/config.py +236 -86
  7. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/debug.py +6 -2
  8. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/instrument.py +133 -79
  9. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/model_cap.py +2 -2
  10. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/otel.py +75 -36
  11. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/providers.py +45 -21
  12. frugal_sdk_python-0.7.0/src/frugal_metrics/safe.py +58 -0
  13. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/semconv.py +8 -1
  14. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/streams.py +5 -4
  15. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/track.py +8 -6
  16. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/_batch.py +20 -12
  17. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/anthropic.py +4 -4
  18. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/bedrock.py +4 -4
  19. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/openai.py +4 -4
  20. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/conftest.py +24 -0
  21. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_config.py +66 -3
  22. frugal_sdk_python-0.7.0/tests/test_node_parity.py +791 -0
  23. frugal_sdk_python-0.7.0/tests/test_otel_temporality.py +171 -0
  24. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_providers.py +31 -0
  25. frugal_sdk_python-0.7.0/tests/test_sibling_parity.py +79 -0
  26. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_anthropic_batches.py +52 -0
  27. frugal_sdk_python-0.5.0/src/frugal_metrics/safe.py +0 -45
  28. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/.github/workflows/cd.yml +0 -0
  29. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/.gitignore +0 -0
  30. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/LICENSES/Apache-2.0.txt +0 -0
  31. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/REUSE.toml +0 -0
  32. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/README.md +0 -0
  33. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/collector/otel-collector-config.yaml +0 -0
  34. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/docker-compose.yml +0 -0
  35. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/grafana/dashboards/frugal-metrics.json +0 -0
  36. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/grafana/provisioning/dashboards/frugal.yml +0 -0
  37. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/grafana/provisioning/datasources/prometheus.yml +0 -0
  38. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/mockllm/Dockerfile +0 -0
  39. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/mockllm/src/server.js +0 -0
  40. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/prometheus/consoles/index.html.example +0 -0
  41. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/prometheus/consoles/models.html +0 -0
  42. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/prometheus/consoles/overview.html +0 -0
  43. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/prometheus/consoles/sites.html +0 -0
  44. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/prometheus/prometheus.yml +0 -0
  45. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/Dockerfile +0 -0
  46. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/__init__.py +0 -0
  47. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/clients.py +0 -0
  48. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/__init__.py +0 -0
  49. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/anon.py +0 -0
  50. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/classify.py +0 -0
  51. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/deferred.py +0 -0
  52. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/fail.py +0 -0
  53. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/pipeline.py +0 -0
  54. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/summarize.py +0 -0
  55. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/translate.py +0 -0
  56. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/via_blocked_lib.py +0 -0
  57. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/handlers/via_unblocked_lib.py +0 -0
  58. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/internal_lib/__init__.py +0 -0
  59. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/internal_lib/ai_helper.py +0 -0
  60. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/main.py +0 -0
  61. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/sibling_lib/__init__.py +0 -0
  62. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/app/sibling_lib/ai_helper.py +0 -0
  63. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/pyproject.toml +0 -0
  64. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/sample-app/requirements.txt +0 -0
  65. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/examples/test-env/scripts/verify.sh +0 -0
  66. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/pyproject.toml +0 -0
  67. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/__init__.py +0 -0
  68. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/block_hash_lru.py +0 -0
  69. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/caller.py +0 -0
  70. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/duplicate_filter.py +0 -0
  71. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/prompt_analyzer.py +0 -0
  72. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/shutdown.py +0 -0
  73. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/__init__.py +0 -0
  74. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/_patching.py +0 -0
  75. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/src/frugal_metrics/wrappers/bedrock_models.py +0 -0
  76. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/__init__.py +0 -0
  77. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_bedrock_models.py +0 -0
  78. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_block_hash_lru.py +0 -0
  79. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_caller.py +0 -0
  80. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_caller_edge_cases.py +0 -0
  81. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_debug.py +0 -0
  82. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_duplicate_filter.py +0 -0
  83. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_model_cap.py +0 -0
  84. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_never_impact_host.py +0 -0
  85. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_otel_overflow_detection.py +0 -0
  86. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_prompt_analyzer.py +0 -0
  87. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_shutdown.py +0 -0
  88. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_track_ai_call.py +0 -0
  89. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_anthropic.py +0 -0
  90. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_bedrock.py +0 -0
  91. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_openai.py +0 -0
  92. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_openai_analyzer.py +0 -0
  93. {frugal_sdk_python-0.5.0 → frugal_sdk_python-0.7.0}/tests/test_wrap_openai_batches.py +0 -0
@@ -1,6 +1,6 @@
1
1
  Metadata-Version: 2.5
2
2
  Name: frugal-sdk-python
3
- Version: 0.5.0
3
+ Version: 0.7.0
4
4
  Summary: Instrumentation for Cost-to-Code attribution.
5
5
  Project-URL: Homepage, https://frugal.co
6
6
  Author: Frugal AI Inc.
@@ -62,7 +62,7 @@ All configuration is via environment variables. No `init()` call, no configurati
62
62
 
63
63
  ### Required
64
64
 
65
- Set both or the wrapper silently returns the client unmodified:
65
+ Console mode (`FRUGAL_METRICS_EXPORTER=console`) needs only `FRUGAL_PROJECT_ID`; every other mode needs both. If a required variable is unset, the wrapper silently returns the client unmodified:
66
66
 
67
67
  | Variable | Purpose |
68
68
  | ------------------- | ---------------------------------------------------------- |
@@ -71,23 +71,33 @@ Set both or the wrapper silently returns the client unmodified:
71
71
 
72
72
  ### Optional
73
73
 
74
+ The export endpoint and authentication come only from `FRUGAL_`-prefixed variables, so the SDK never crosses wires with the host's own OTel pipeline: the generic `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` are never read. (The OTel exporter library still reads its own generic TLS, compression and timeout variables.)
75
+
76
+ Export temporality is pinned by the SDK: Counter, Histogram and ObservableCounter instruments are exported as OTLP **delta**, everything else as cumulative. `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` is ignored, and every export carries the resource attribute `frugal.temporality="delta"`.
77
+
78
+ Delta delivery is best-effort: the exporter retries transient failures (throttling, gateway errors, network errors). An interval whose export still fails after those retries (for example a long outage), or fails with a non-retryable error such as a rejected token, is dropped, not carried into the next export. A retry after a lost response can occasionally deliver an interval twice. Each run of failed exports surfaces once through the `error` debug callback.
79
+
80
+ **Querying:** use `sum_over_time` / `rate_over_sum`, not `increase` / `rate`, and select `frugal.temporality="delta"`.
81
+
74
82
  | Variable | Purpose |
75
83
  | ---------------------------------------- | --------------------------------------------------------------------------------------- |
76
84
  | `FRUGAL_CUSTOMER_ID` | Customer ID issued by Frugal. When unset, the tenant label (auto-injected server-side by VMAuth from the bearer token) identifies the customer and `frugal.customer_id` is omitted. |
77
85
  | `FRUGAL_REPO_ROOT` | Absolute path to the source root; the strip prefix for `caller_file`. Defaults to the process working directory. |
78
- | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector URL for Frugal metrics. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics`. Falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`. |
79
- | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Overrides the header built from `FRUGAL_API_KEY`. Falls back to `OTEL_EXPORTER_OTLP_HEADERS`. |
86
+ | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector URL for Frugal metrics. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics` (also when set but empty). |
87
+ | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Precedence: this variable wins; otherwise the header built from `FRUGAL_API_KEY` (`FRUGAL_METRICS_API_KEY` first), which is sent verbatim. Values set here are percent-decoded (`Bearer%20tok` → `Bearer tok`). A value with no `key=value` pair is treated as missing auth (the API key is not used instead); a blank (whitespace-only) value is treated as unset. |
80
88
  | `FRUGAL_COMPONENTS` | Comma-separated logical components (e.g. `ai-handler,reco`). When set, each metric is ALSO emitted once per component (tagged `frugal.component`) on top of the base project series, so usage can be viewed for the whole project or split by component. Deduped, and only the first **10** entries are honored (hard cap, not configurable); the rest are dropped at startup with one debug warning. |
81
89
  | `FRUGAL_CUSTOM_PROVIDERS` | Comma-separated allowlist of provider names beyond the built-in registry (e.g. `together_ai,fireworks`) that should pass through verbatim on `gen_ai.system` / `gen_ai.provider.name` instead of being coerced to `custom`. Capped at 10 (extras are coerced) to keep label cardinality bounded. |
82
- | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; a value outside the first 20 entries (hard cap, not configurable) is coerced to `other`. |
90
+ | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`. The vocabulary is the first **20** comma-separated entries by raw position (hard cap, not configurable); empty entries are ignored and duplicates collapsed. `other` is reserved (the coercion sentinel) and is ignored with one warning. A value outside the allowlist is coerced to `other` with a bounded once-per-value warning; when the variable is unset, every value is `other`. One warning fires only when non-empty entries past the 20th are actually dropped (a trailing comma does not warn). |
91
+ | `FRUGAL_METRICS_EXPORTER` | `otlp` (default) or `console` (print metrics to stdout; console mode needs only `FRUGAL_PROJECT_ID`; the API key, endpoint and header variables are ignored). Case-insensitive; any other value warns once and uses `otlp`. |
92
+ | `FRUGAL_METRICS_PROJECT_ID` / `FRUGAL_METRICS_API_KEY` | Namespaced overrides read before `FRUGAL_PROJECT_ID` / `FRUGAL_API_KEY`, for apps that already use those names. An empty value falls through to the bare name. |
83
93
  | `FRUGAL_PATH_BLOCK_LIST` | Comma-separated repo-relative paths to skip for caller attribution (see below). |
84
- | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence in milliseconds. Default `60000`. |
94
+ | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence in milliseconds, `1`-`2147483647`. Default `60000`; a value outside the range warns once and uses the default. |
85
95
  | `FRUGAL_MAX_CALL_SITES` | Cardinality cap — max distinct `(caller_file, caller_function)` pairs to attribute; past it `frugal.caller_file` / `frugal.caller_function` become `<overflow>`. Default `500`. Set to `0` to disable. |
86
96
  | `FRUGAL_MAX_MODELS` | Cardinality cap — max distinct model strings to attribute; past it `gen_ai.request.model` / `gen_ai.response.model` become `<overflow>`. One registry shared by both labels, so a request alias and its dated response snapshot each take a slot. Default `100`. Set to `0` to disable. |
87
97
  | `FRUGAL_MAX_SERIES_PER_INSTRUMENT` | OpenTelemetry's per-instrument limit on distinct attribute **combinations**, raised from the SDK default. Default `50000`. Set to `0` for effectively unlimited (a 1,000,000 ceiling). Applied **only when the installed `opentelemetry-sdk` accepts a per-instrument cardinality limit** — current releases (through 1.42) do not implement one, so nothing is capped and setting this variable emits one `warn` telling you it has no effect. |
88
- | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls (0..1) that run the lightweight prompt analyzer. Default `0.1`. Set to `0` to disable, `1` for every call. |
98
+ | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls that run the lightweight prompt analyzer: a decimal in [0, 1], plain or scientific notation (e.g. `0.25`, `1e-1`); anything else warns once and uses `0.1`. Default `0.1`. Set to `0` to disable, `1` for every call. |
89
99
  | `FRUGAL_DUPLICATE_REQUEST_DETECTION` | Duplicate-request detection (the `frugal.gen_ai.duplicate_requests` metric) is **on by default** and hashes each chat request in-process (the hash never leaves the SDK). Set to `0`/`false`/`no`/`off` to disable — no hashing, no filter memory, no metric. |
90
- | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `604800` (7 days). |
100
+ | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `86400` (1 day). |
91
101
  | `FRUGAL_DUPLICATE_CAPACITY` | Expected distinct request signatures per window (sizes the in-memory filter). Default `10000`, max `1000000`. Raise it if the SDK warns the filter is saturated. |
92
102
  | `FRUGAL_DUPLICATE_FP_RATE` | Target false-positive rate of the duplicate filter. Default `0.01`, clamped to `[1e-6, 0.5]`. |
93
103
  | `FRUGAL_DUPLICATE_GENERATIONS` | Rotation granularity of the duplicate filter's time window. Default `2`, max `16`. |
@@ -125,7 +135,8 @@ Every metric carries this base attribute set (except
125
135
  `frugal.gen_ai.internal_errors`, which is emitted from failure paths where
126
136
  these attributes may not exist yet and carries only `frugal.project_id`, the
127
137
  stage/type, `frugal.instrument` on the `otel_cardinality_overflow` stage, and
128
- `frugal.customer_id` / `frugal.custom_label` when set):
138
+ `frugal.customer_id` / `frugal.custom_label` when set; `frugal.custom_label`
139
+ never on the setup_failed / emission stages):
129
140
 
130
141
  - `frugal.project_id` — from your env vars (`frugal.customer_id` too when set); `frugal.component` is added on the per-component series when `FRUGAL_COMPONENTS` is set
131
142
  - `frugal.caller_file`, `frugal.caller_function` — auto-detected from the call stack
@@ -133,7 +144,7 @@ stage/type, `frugal.instrument` on the `otel_cardinality_overflow` stage, and
133
144
  - `gen_ai.request.model`, `gen_ai.response.model` — the model used; both draw on one shared registry and become `<overflow>` past `FRUGAL_MAX_MODELS` distinct model strings
134
145
  - `gen_ai.operation.name` — `chat`, `text_completion`, `embeddings`, `generate_content`, `execute_tool`, `create_agent`, `invoke_agent`, or `other` (the sentinel an unrecognized operation coerces to)
135
146
  - `frugal.batched` — `true` for batch-API paths (OpenAI Batch API submissions; Anthropic Message Batches submissions and results), `false` otherwise
136
- - `frugal.custom_label` — only on `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; the value must be in the `FRUGAL_CUSTOM_LABEL` allowlist or it is coerced to `other`
147
+ - `frugal.custom_label` — only on `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; the value must be in the `FRUGAL_CUSTOM_LABEL` allowlist or it is coerced to `other` (with a once-per-value warning). `custom_label=None` means no label; a non-string value is coerced to `other`
137
148
 
138
149
  The "Additional attributes" column above lists what each metric carries on top of the base set. See [`docs/metrics-reference.md`](docs/metrics-reference.md) for the values each attribute takes, sentinel values for caller fields, and the full internal-error-stage list.
139
150
 
@@ -228,7 +239,7 @@ except BaseException as exc:
228
239
 
229
240
  `success()` and `error()` finalize the call exactly once and are idempotent — extra calls are ignored.
230
241
 
231
- **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`), `custom_label` (→ the `frugal.custom_label` label on every metric of the call; trimmed, and coerced to `other` unless in the `FRUGAL_CUSTOM_LABEL` allowlist). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
242
+ **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`), `custom_label` (→ the `frugal.custom_label` label on every metric of the call except the `frugal.gen_ai.internal_errors` setup_failed / emission points; trimmed, and coerced to `other` unless in the `FRUGAL_CUSTOM_LABEL` allowlist). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
232
243
 
233
244
  **Usage** — `set_usage(...)` / `success(...)` kwargs (all optional): `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `agent_turns`, `response_model`.
234
245
 
@@ -323,18 +334,33 @@ enable_auto_shutdown() # flush + shutdown via atexit, plus SIGTERM / SIGINT
323
334
 
324
335
  `enable_auto_shutdown` always registers an `atexit` hook and, when `handle_signals=True` (the default), installs SIGTERM/SIGINT handlers that flush+shutdown and then **chain to whatever handler was previously installed** (so your own handler still runs). Off the main thread, signal registration is unavailable and it falls back to atexit-only. It is **idempotent** — repeat calls register nothing extra.
325
336
 
337
+ Exports are delta: each carries only its own interval, so a skipped final flush loses at most the last interval.
338
+
326
339
  It is **opt-in** because installing signal handlers is intrusive. If you run your own signal handling, do **not** call `enable_auto_shutdown(handle_signals=True)`; prefer calling `shutdown()` from inside your own handler instead.
327
340
 
328
341
  ## Verifying the setup
329
342
 
330
- Set the OTLP endpoint to empty to print metrics to the console instead of exporting them:
343
+ Set `FRUGAL_METRICS_EXPORTER=console` to print the metrics to stdout instead
344
+ of exporting them. Console mode needs only `FRUGAL_PROJECT_ID`; the API key,
345
+ endpoint and header variables are ignored. An empty
346
+ endpoint does not select the console exporter: it falls through to the default
347
+ Frugal endpoint.
348
+
349
+ ```bash
350
+ export FRUGAL_PROJECT_ID=my-project
351
+ export FRUGAL_METRICS_EXPORTER=console
352
+ export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
353
+ python my_script.py
354
+ ```
355
+
356
+ Or point the library at a local OTLP/HTTP collector to see what it exports:
331
357
 
332
358
  ```bash
333
- export FRUGAL_API_KEY=local
359
+ export FRUGAL_API_KEY=test-key
334
360
  export FRUGAL_PROJECT_ID=my-project
335
- export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT= # empty → ConsoleMetricExporter
361
+ export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/metrics
336
362
  export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
337
363
  python my_script.py
338
364
  ```
339
365
 
340
- You should see JSON metric records with `gen_ai.*` and `frugal.*` attributes every 5 seconds.
366
+ The collector should receive metric records with `gen_ai.*` and `frugal.*` attributes on each 5-second export that follows at least one wrapped call. Counters and histograms are exported as delta, so an idle interval carries no counter or histogram points: make a call, then watch the next export.
@@ -33,7 +33,7 @@ All configuration is via environment variables. No `init()` call, no configurati
33
33
 
34
34
  ### Required
35
35
 
36
- Set both or the wrapper silently returns the client unmodified:
36
+ Console mode (`FRUGAL_METRICS_EXPORTER=console`) needs only `FRUGAL_PROJECT_ID`; every other mode needs both. If a required variable is unset, the wrapper silently returns the client unmodified:
37
37
 
38
38
  | Variable | Purpose |
39
39
  | ------------------- | ---------------------------------------------------------- |
@@ -42,23 +42,33 @@ Set both or the wrapper silently returns the client unmodified:
42
42
 
43
43
  ### Optional
44
44
 
45
+ The export endpoint and authentication come only from `FRUGAL_`-prefixed variables, so the SDK never crosses wires with the host's own OTel pipeline: the generic `OTEL_EXPORTER_OTLP_ENDPOINT` and `OTEL_EXPORTER_OTLP_HEADERS` are never read. (The OTel exporter library still reads its own generic TLS, compression and timeout variables.)
46
+
47
+ Export temporality is pinned by the SDK: Counter, Histogram and ObservableCounter instruments are exported as OTLP **delta**, everything else as cumulative. `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` is ignored, and every export carries the resource attribute `frugal.temporality="delta"`.
48
+
49
+ Delta delivery is best-effort: the exporter retries transient failures (throttling, gateway errors, network errors). An interval whose export still fails after those retries (for example a long outage), or fails with a non-retryable error such as a rejected token, is dropped, not carried into the next export. A retry after a lost response can occasionally deliver an interval twice. Each run of failed exports surfaces once through the `error` debug callback.
50
+
51
+ **Querying:** use `sum_over_time` / `rate_over_sum`, not `increase` / `rate`, and select `frugal.temporality="delta"`.
52
+
45
53
  | Variable | Purpose |
46
54
  | ---------------------------------------- | --------------------------------------------------------------------------------------- |
47
55
  | `FRUGAL_CUSTOMER_ID` | Customer ID issued by Frugal. When unset, the tenant label (auto-injected server-side by VMAuth from the bearer token) identifies the customer and `frugal.customer_id` is omitted. |
48
56
  | `FRUGAL_REPO_ROOT` | Absolute path to the source root; the strip prefix for `caller_file`. Defaults to the process working directory. |
49
- | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector URL for Frugal metrics. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics`. Falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`. |
50
- | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Overrides the header built from `FRUGAL_API_KEY`. Falls back to `OTEL_EXPORTER_OTLP_HEADERS`. |
57
+ | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | OTLP collector URL for Frugal metrics. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics` (also when set but empty). |
58
+ | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Precedence: this variable wins; otherwise the header built from `FRUGAL_API_KEY` (`FRUGAL_METRICS_API_KEY` first), which is sent verbatim. Values set here are percent-decoded (`Bearer%20tok` → `Bearer tok`). A value with no `key=value` pair is treated as missing auth (the API key is not used instead); a blank (whitespace-only) value is treated as unset. |
51
59
  | `FRUGAL_COMPONENTS` | Comma-separated logical components (e.g. `ai-handler,reco`). When set, each metric is ALSO emitted once per component (tagged `frugal.component`) on top of the base project series, so usage can be viewed for the whole project or split by component. Deduped, and only the first **10** entries are honored (hard cap, not configurable); the rest are dropped at startup with one debug warning. |
52
60
  | `FRUGAL_CUSTOM_PROVIDERS` | Comma-separated allowlist of provider names beyond the built-in registry (e.g. `together_ai,fireworks`) that should pass through verbatim on `gen_ai.system` / `gen_ai.provider.name` instead of being coerced to `custom`. Capped at 10 (extras are coerced) to keep label cardinality bounded. |
53
- | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; a value outside the first 20 entries (hard cap, not configurable) is coerced to `other`. |
61
+ | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`. The vocabulary is the first **20** comma-separated entries by raw position (hard cap, not configurable); empty entries are ignored and duplicates collapsed. `other` is reserved (the coercion sentinel) and is ignored with one warning. A value outside the allowlist is coerced to `other` with a bounded once-per-value warning; when the variable is unset, every value is `other`. One warning fires only when non-empty entries past the 20th are actually dropped (a trailing comma does not warn). |
62
+ | `FRUGAL_METRICS_EXPORTER` | `otlp` (default) or `console` (print metrics to stdout; console mode needs only `FRUGAL_PROJECT_ID`; the API key, endpoint and header variables are ignored). Case-insensitive; any other value warns once and uses `otlp`. |
63
+ | `FRUGAL_METRICS_PROJECT_ID` / `FRUGAL_METRICS_API_KEY` | Namespaced overrides read before `FRUGAL_PROJECT_ID` / `FRUGAL_API_KEY`, for apps that already use those names. An empty value falls through to the bare name. |
54
64
  | `FRUGAL_PATH_BLOCK_LIST` | Comma-separated repo-relative paths to skip for caller attribution (see below). |
55
- | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence in milliseconds. Default `60000`. |
65
+ | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence in milliseconds, `1`-`2147483647`. Default `60000`; a value outside the range warns once and uses the default. |
56
66
  | `FRUGAL_MAX_CALL_SITES` | Cardinality cap — max distinct `(caller_file, caller_function)` pairs to attribute; past it `frugal.caller_file` / `frugal.caller_function` become `<overflow>`. Default `500`. Set to `0` to disable. |
57
67
  | `FRUGAL_MAX_MODELS` | Cardinality cap — max distinct model strings to attribute; past it `gen_ai.request.model` / `gen_ai.response.model` become `<overflow>`. One registry shared by both labels, so a request alias and its dated response snapshot each take a slot. Default `100`. Set to `0` to disable. |
58
68
  | `FRUGAL_MAX_SERIES_PER_INSTRUMENT` | OpenTelemetry's per-instrument limit on distinct attribute **combinations**, raised from the SDK default. Default `50000`. Set to `0` for effectively unlimited (a 1,000,000 ceiling). Applied **only when the installed `opentelemetry-sdk` accepts a per-instrument cardinality limit** — current releases (through 1.42) do not implement one, so nothing is capped and setting this variable emits one `warn` telling you it has no effect. |
59
- | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls (0..1) that run the lightweight prompt analyzer. Default `0.1`. Set to `0` to disable, `1` for every call. |
69
+ | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls that run the lightweight prompt analyzer: a decimal in [0, 1], plain or scientific notation (e.g. `0.25`, `1e-1`); anything else warns once and uses `0.1`. Default `0.1`. Set to `0` to disable, `1` for every call. |
60
70
  | `FRUGAL_DUPLICATE_REQUEST_DETECTION` | Duplicate-request detection (the `frugal.gen_ai.duplicate_requests` metric) is **on by default** and hashes each chat request in-process (the hash never leaves the SDK). Set to `0`/`false`/`no`/`off` to disable — no hashing, no filter memory, no metric. |
61
- | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `604800` (7 days). |
71
+ | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `86400` (1 day). |
62
72
  | `FRUGAL_DUPLICATE_CAPACITY` | Expected distinct request signatures per window (sizes the in-memory filter). Default `10000`, max `1000000`. Raise it if the SDK warns the filter is saturated. |
63
73
  | `FRUGAL_DUPLICATE_FP_RATE` | Target false-positive rate of the duplicate filter. Default `0.01`, clamped to `[1e-6, 0.5]`. |
64
74
  | `FRUGAL_DUPLICATE_GENERATIONS` | Rotation granularity of the duplicate filter's time window. Default `2`, max `16`. |
@@ -96,7 +106,8 @@ Every metric carries this base attribute set (except
96
106
  `frugal.gen_ai.internal_errors`, which is emitted from failure paths where
97
107
  these attributes may not exist yet and carries only `frugal.project_id`, the
98
108
  stage/type, `frugal.instrument` on the `otel_cardinality_overflow` stage, and
99
- `frugal.customer_id` / `frugal.custom_label` when set):
109
+ `frugal.customer_id` / `frugal.custom_label` when set; `frugal.custom_label`
110
+ never on the setup_failed / emission stages):
100
111
 
101
112
  - `frugal.project_id` — from your env vars (`frugal.customer_id` too when set); `frugal.component` is added on the per-component series when `FRUGAL_COMPONENTS` is set
102
113
  - `frugal.caller_file`, `frugal.caller_function` — auto-detected from the call stack
@@ -104,7 +115,7 @@ stage/type, `frugal.instrument` on the `otel_cardinality_overflow` stage, and
104
115
  - `gen_ai.request.model`, `gen_ai.response.model` — the model used; both draw on one shared registry and become `<overflow>` past `FRUGAL_MAX_MODELS` distinct model strings
105
116
  - `gen_ai.operation.name` — `chat`, `text_completion`, `embeddings`, `generate_content`, `execute_tool`, `create_agent`, `invoke_agent`, or `other` (the sentinel an unrecognized operation coerces to)
106
117
  - `frugal.batched` — `true` for batch-API paths (OpenAI Batch API submissions; Anthropic Message Batches submissions and results), `false` otherwise
107
- - `frugal.custom_label` — only on `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; the value must be in the `FRUGAL_CUSTOM_LABEL` allowlist or it is coerced to `other`
118
+ - `frugal.custom_label` — only on `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; the value must be in the `FRUGAL_CUSTOM_LABEL` allowlist or it is coerced to `other` (with a once-per-value warning). `custom_label=None` means no label; a non-string value is coerced to `other`
108
119
 
109
120
  The "Additional attributes" column above lists what each metric carries on top of the base set. See [`docs/metrics-reference.md`](docs/metrics-reference.md) for the values each attribute takes, sentinel values for caller fields, and the full internal-error-stage list.
110
121
 
@@ -199,7 +210,7 @@ except BaseException as exc:
199
210
 
200
211
  `success()` and `error()` finalize the call exactly once and are idempotent — extra calls are ignored.
201
212
 
202
- **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`), `custom_label` (→ the `frugal.custom_label` label on every metric of the call; trimmed, and coerced to `other` unless in the `FRUGAL_CUSTOM_LABEL` allowlist). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
213
+ **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional): `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default `"chat"`), `request_model` (default `"<unknown>"`), `custom_label` (→ the `frugal.custom_label` label on every metric of the call except the `frugal.gen_ai.internal_errors` setup_failed / emission points; trimmed, and coerced to `other` unless in the `FRUGAL_CUSTOM_LABEL` allowlist). To also run the sampled prompt analyzer, pass any of `messages` / `system_prompt` / `tools` / `response_format` / `max_tokens`.
203
214
 
204
215
  **Usage** — `set_usage(...)` / `success(...)` kwargs (all optional): `input_tokens`, `output_tokens`, `cache_read_tokens`, `cache_write_tokens`, `reasoning_tokens`, `agent_turns`, `response_model`.
205
216
 
@@ -294,18 +305,33 @@ enable_auto_shutdown() # flush + shutdown via atexit, plus SIGTERM / SIGINT
294
305
 
295
306
  `enable_auto_shutdown` always registers an `atexit` hook and, when `handle_signals=True` (the default), installs SIGTERM/SIGINT handlers that flush+shutdown and then **chain to whatever handler was previously installed** (so your own handler still runs). Off the main thread, signal registration is unavailable and it falls back to atexit-only. It is **idempotent** — repeat calls register nothing extra.
296
307
 
308
+ Exports are delta: each carries only its own interval, so a skipped final flush loses at most the last interval.
309
+
297
310
  It is **opt-in** because installing signal handlers is intrusive. If you run your own signal handling, do **not** call `enable_auto_shutdown(handle_signals=True)`; prefer calling `shutdown()` from inside your own handler instead.
298
311
 
299
312
  ## Verifying the setup
300
313
 
301
- Set the OTLP endpoint to empty to print metrics to the console instead of exporting them:
314
+ Set `FRUGAL_METRICS_EXPORTER=console` to print the metrics to stdout instead
315
+ of exporting them. Console mode needs only `FRUGAL_PROJECT_ID`; the API key,
316
+ endpoint and header variables are ignored. An empty
317
+ endpoint does not select the console exporter: it falls through to the default
318
+ Frugal endpoint.
319
+
320
+ ```bash
321
+ export FRUGAL_PROJECT_ID=my-project
322
+ export FRUGAL_METRICS_EXPORTER=console
323
+ export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
324
+ python my_script.py
325
+ ```
326
+
327
+ Or point the library at a local OTLP/HTTP collector to see what it exports:
302
328
 
303
329
  ```bash
304
- export FRUGAL_API_KEY=local
330
+ export FRUGAL_API_KEY=test-key
305
331
  export FRUGAL_PROJECT_ID=my-project
306
- export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT= # empty → ConsoleMetricExporter
332
+ export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/metrics
307
333
  export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
308
334
  python my_script.py
309
335
  ```
310
336
 
311
- You should see JSON metric records with `gen_ai.*` and `frugal.*` attributes every 5 seconds.
337
+ The collector should receive metric records with `gen_ai.*` and `frugal.*` attributes on each 5-second export that follows at least one wrapped call. Counters and histograms are exported as delta, so an idle interval carries no counter or histogram points: make a call, then watch the next export.
@@ -61,9 +61,9 @@ combining both would double-count tokens.
61
61
  To derive average turns per reporting agent invocation in MetricsQL, use:
62
62
 
63
63
  ```promql
64
- increase(frugal.gen_ai.agent.turns_sum[...])
64
+ sum_over_time(frugal.gen_ai.agent.turns_sum{frugal.temporality="delta"}[...])
65
65
  /
66
- increase(frugal.gen_ai.agent.turns_count[...])
66
+ sum_over_time(frugal.gen_ai.agent.turns_count{frugal.temporality="delta"}[...])
67
67
  ```
68
68
 
69
69
  The analyzer-sampled metrics are gated by `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE`
@@ -120,8 +120,8 @@ Controls:
120
120
  `false`/`0`/`no`/`off`, no filter is allocated, no signature is hashed, and the
121
121
  metric is not emitted. Any other value keeps detection enabled and emits a
122
122
  one-time `warn` diagnostic naming the unrecognized value.
123
- - `FRUGAL_DUPLICATE_WINDOW_SECONDS` (default `3600`, clamped to `604800` — 7
124
- days) — retention window. Because the filter rotates in `generations` slots,
123
+ - `FRUGAL_DUPLICATE_WINDOW_SECONDS` (default `3600`, clamped to `86400` — 1
124
+ day) — retention window. Because the filter rotates in `generations` slots,
125
125
  a signature is remembered for **at least**
126
126
  `window × (generations−1)/generations` and **at most** `window` seconds
127
127
  (30–60 min at defaults) — treat the counterfactual cache TTL as that range,
@@ -133,6 +133,15 @@ Controls:
133
133
  - `FRUGAL_DUPLICATE_GENERATIONS` (default `2`, clamped to `16`) — rotating-Bloom
134
134
  generation count.
135
135
 
136
+ Exporter selection: `FRUGAL_METRICS_EXPORTER` (`otlp` default, or `console` to
137
+ print every metric to stdout via the OpenTelemetry console exporter; console
138
+ mode needs only `FRUGAL_PROJECT_ID`; the API key, endpoint and header variables
139
+ are ignored, and any other value warns once and uses `otlp`). An empty endpoint variable falls through to the next source and
140
+ never selects console.
141
+
142
+ `FRUGAL_METRIC_EXPORT_INTERVAL_MS` accepts `1` to `2147483647` (default `60000`);
143
+ a value outside that range warns once and uses the default.
144
+
136
145
  Out-of-range or non-finite values for any of these fall back to the default (or
137
146
  clamp to the bound) with a one-time named `warn` through the debug callback.
138
147
 
@@ -156,6 +165,16 @@ Reading `duplicate_requests` correctly:
156
165
  otherwise double every `calls` series; keeping it separate keeps `calls`
157
166
  stable when detection is toggled.
158
167
 
168
+ ## Temporality
169
+
170
+ Export temporality is pinned by the SDK: Counter, Histogram and ObservableCounter instruments are exported as OTLP **delta**, everything else as cumulative. `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` is ignored, and every export carries the resource attribute `frugal.temporality="delta"`.
171
+
172
+ Delta delivery is best-effort: the exporter retries transient failures (throttling, gateway errors, network errors). An interval whose export still fails after those retries (for example a long outage), or fails with a non-retryable error such as a rejected token, is dropped, not carried into the next export. A retry after a lost response can occasionally deliver an interval twice. Each run of failed exports surfaces once through the `error` debug callback.
173
+
174
+ **Querying:** use `sum_over_time` / `rate_over_sum`, not `increase` / `rate`, and select `frugal.temporality="delta"`.
175
+
176
+ Each export carries only its own interval, so a process that exits without `shutdown()` (or `enable_auto_shutdown()`) loses at most its last interval.
177
+
159
178
  ## Attributes
160
179
 
161
180
  ### From the GenAI semantic convention
@@ -191,18 +210,19 @@ Reading `duplicate_requests` correctly:
191
210
 
192
211
  | Attribute | Value |
193
212
  | ------------------------- | ---------------------------------------------------------- |
194
- | `frugal.customer_id` | From `FRUGAL_CUSTOMER_ID` |
195
- | `frugal.project_id` | From `FRUGAL_PROJECT_ID` |
213
+ | `frugal.customer_id` | From `FRUGAL_CUSTOMER_ID` — present only when set |
214
+ | `frugal.project_id` | From `FRUGAL_METRICS_PROJECT_ID`, else `FRUGAL_PROJECT_ID` |
215
+ | `frugal.temporality` | Constant resource attribute `delta`, stamped on every export. Select it in queries to skip older cumulative samples that share the same series. |
196
216
  | `frugal.component` | From each entry in `FRUGAL_COMPONENTS` — present only on the per-component series, absent on the base project series |
197
217
  | `frugal.caller_file` | Source file repo-relative to `FRUGAL_REPO_ROOT` |
198
218
  | `frugal.caller_function` | Function/method name at the call site |
199
- | `frugal.error.type` | Exception class name (error-path metrics only) |
200
- | `frugal.batched` | `true` when the metric came from a provider batch-API path, `false` otherwise. Present on every metric. Batch submissions (OpenAI `batches.create`, Anthropic `messages.batches.create`) emit `frugal.gen_ai.calls` with `batched="true"` — one point per request for Anthropic (the requests array is in hand, grouped by model), one per submission for OpenAI (per-request counts are not reported at submission time, so OpenAI batched call volume is a floor, never an overcount). Iterating Anthropic `messages.batches.results()` additionally emits per-entry `gen_ai.client.token.usage` (+ cache metrics) with `batched="true"`, attributed to the call site that submitted the batch when the submission happened in the same process (bounded in-process memory; falls back to the results call site). OpenAI batch token usage lives in the output file read through the generic files API and is not intercepted. The manual track API accepts a `batched` flag for hand-instrumented batch paths. An app on an SDK predating batch support reads `false` on genuinely batched traffic — the label reflects what the deployed SDK can see. |
219
+ | `frugal.error.type` | Exception class name (error-path metrics only); stripped from `frugal.gen_ai.history_depth` and `frugal.gen_ai.duplicate_requests` so success and error points stay additive |
220
+ | `frugal.batched` | `true` when the metric came from a provider batch-API path, `false` otherwise. Present on every metric of the call except the `frugal.gen_ai.internal_errors` points. Batch submissions (OpenAI `batches.create`, Anthropic `messages.batches.create`) emit `frugal.gen_ai.calls` with `batched="true"` — one point per request for Anthropic (the requests array is in hand, grouped by model), one per submission for OpenAI (per-request counts are not reported at submission time, so OpenAI batched call volume is a floor, never an overcount). Iterating Anthropic `messages.batches.results()` additionally emits per-entry `gen_ai.client.token.usage` (+ cache metrics) with `batched="true"`, attributed to the call site that submitted the batch when the submission happened in the same process (bounded in-process memory; falls back to the results call site). OpenAI batch token usage lives in the output file read through the generic files API and is not intercepted. The manual track API accepts a `batched` flag for hand-instrumented batch paths. An app on an SDK predating batch support reads `false` on genuinely batched traffic — the label reflects what the deployed SDK can see. |
201
221
  | `frugal.structured_output_mode` | `none` \| `json_object` \| `json_schema` \| `tools`. Derived from request params by precedence: non-empty `tools` → `tools`; else `response_format.type` `json_object` / `json_schema` → the matching value; else `none`. On `frugal.gen_ai.output_cap_utilization` (when `max_tokens` set), and on `frugal.gen_ai.calls` for every call with a request payload (omitted when there is no payload). |
202
222
  | `frugal.block` | `system` \| `user` \| `tools` \| `schema`. Only on `frugal.gen_ai.cache_prefix_stable`. |
203
223
  | `frugal.stable` | `true` when the block hash matches the previous call at this site. Only on `frugal.gen_ai.cache_prefix_stable`. |
204
224
  | `frugal.history_bucket` | Bounded bucket of `messages.length`: `0` \| `1-2` \| `3-5` \| `6-10` \| `11-20` \| `21-50` \| `51+`. On `frugal.gen_ai.calls` and `gen_ai.client.token.usage`, and only for chat calls (payload has a `messages` array); omitted otherwise. 100% (not analyzer-sampled). |
205
- | `frugal.custom_label` | Custom label (e.g. a business unit) passed as `custom_label=` to `track_ai_call` / `start_ai_call` — never set by the provider wrappers. When passed, it rides EVERY metric of that call (success and error paths alike). The value is trimmed of surrounding whitespace (matching the Node SDK), then must match one of the first 20 entries of `FRUGAL_CUSTOM_LABEL` (hard cap, not configurable) or it is coerced to `other`. Omitted when not passed. |
225
+ | `frugal.custom_label` | Custom label (e.g. a business unit) passed as `custom_label=` to `track_ai_call` / `start_ai_call` — never set by the provider wrappers. When passed, it rides every metric of that call (success and error paths alike), except the `frugal.gen_ai.internal_errors` `setup_failed`/`emission` points, which carry only project, customer and error type. The value is trimmed of surrounding whitespace (matching the Node SDK), then must match the closed vocabulary from the first 20 comma-separated entries of `FRUGAL_CUSTOM_LABEL` by raw position (hard cap, not configurable; empties ignored, duplicates collapsed) or it is coerced to `other` with a bounded once-per-value warning. `other` is reserved as the coercion sentinel: an `other` entry in the env var is ignored with a warning. With the variable unset, every value is `other`. A non-string value is coerced to `other` with a type warning; `None` means no label. A truncation warning fires only when non-empty entries past the 20th are dropped. Omitted when not passed. |
206
226
  | `frugal.dup_state` | `first_seen` \| `repeat_seen` — whether this request's canonical signature was already seen within the duplicate-detection window. Only on `frugal.gen_ai.duplicate_requests`. |
207
227
 
208
228
  ### Sentinel values for `frugal.caller_file` / `frugal.caller_function`
@@ -257,9 +277,10 @@ attribute describing where our own code failed. Non-exhaustive list:
257
277
 
258
278
  | Stage | When it fires |
259
279
  | ----------------------- | ---------------------------------------------------------------------- |
260
- | `setup_failed` | Our wrapper setup (OTel init, caller resolution) threw; original call continued normally |
280
+ | `setup_failed` | Our wrapper setup (OTel init, caller resolution) threw; original call continued normally. Emitted by the provider wrappers and the manual track API (`track_ai_call` / `start_ai_call`); carries `frugal.project_id`, `frugal.customer_id` (when set) and `frugal.internal_error.type` |
261
281
  | `stream_proxy_failed` | Constructing the stream wrapper threw; original response returned unwrapped |
262
- | `emission` | Emitting a success/error metric threw |
282
+ | `emission` | Emitting a success/error metric threw. Emitted by the provider wrappers and the manual track API; carries the same project/customer identity and `frugal.internal_error.type` as `setup_failed` |
283
+ | `call_shape` | Deriving history_depth / history_bucket / structured_output_mode from the request payload threw; carries `frugal.internal_error.type`; core metrics still emitted |
263
284
  | `duplicate_detection` | The duplicate-request path (signature, filter, or emission) threw — including the one-time filter allocation failure; carries `frugal.internal_error.type` with the exception class |
264
285
  | `cardinality_cap_hit` | `FRUGAL_MAX_CALL_SITES` was just reached (once per process) |
265
286
  | `model_cap_hit` | `FRUGAL_MAX_MODELS` was just reached (once per process); further distinct models are reported as `<overflow>` |
@@ -271,7 +292,8 @@ caller/provider/model attributes may not exist yet. Its points carry only
271
292
  `frugal.project_id`, `frugal.internal_error.stage` (plus
272
293
  `frugal.internal_error.type` where noted, and `frugal.instrument` on the
273
294
  `otel_cardinality_overflow` stage), and — when set — `frugal.customer_id` and
274
- `frugal.custom_label`.
295
+ `frugal.custom_label` (`setup_failed` and `emission` never carry the custom
296
+ label).
275
297
 
276
298
  ## Provider coverage
277
299
 
@@ -10,39 +10,48 @@ pip install openai anthropic
10
10
 
11
11
  ## Configure (env vars only)
12
12
 
13
- Required — the library silently no-ops if either is unset:
13
+ Required — console mode (`FRUGAL_METRICS_EXPORTER=console`) needs only `FRUGAL_PROJECT_ID`; every other mode needs both. The library silently no-ops if a required variable is unset:
14
14
 
15
15
  | Var | Value |
16
16
  | ------------------- | ------------------------------------------------------------------- |
17
17
  | `FRUGAL_API_KEY` | Per-customer bearer token issued by Frugal; builds the export `Authorization` header. (Or supply the bearer yourself via `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS`.) |
18
18
  | `FRUGAL_PROJECT_ID` | The Frugal project this deployment belongs to. Stats are attributed per project, so a shared repo across microservices no longer collides. |
19
19
 
20
- Optional:
20
+ Optional — the export endpoint and authentication come only from
21
+ `FRUGAL_`-prefixed variables, so the SDK never crosses wires with the host's
22
+ own OTel pipeline: the generic `OTEL_EXPORTER_OTLP_ENDPOINT` and
23
+ `OTEL_EXPORTER_OTLP_HEADERS` are never read. (The OTel exporter library still
24
+ reads its own generic TLS, compression and timeout variables.)
25
+
26
+ Export temporality is pinned by the SDK: Counter, Histogram and ObservableCounter instruments are exported as OTLP **delta**, everything else as cumulative. `OTEL_EXPORTER_OTLP_METRICS_TEMPORALITY_PREFERENCE` is ignored, and every export carries the resource attribute `frugal.temporality="delta"`.
27
+
28
+ Delta delivery is best-effort: the exporter retries transient failures (throttling, gateway errors, network errors). An interval whose export still fails after those retries (for example a long outage), or fails with a non-retryable error such as a rejected token, is dropped, not carried into the next export. A retry after a lost response can occasionally deliver an interval twice. Each run of failed exports surfaces once through the `error` debug callback.
29
+
30
+ **Querying:** use `sum_over_time` / `rate_over_sum`, not `increase` / `rate`, and select `frugal.temporality="delta"`.
21
31
 
22
32
  | Var | Purpose |
23
33
  | ------------------------------------- | ------------------------------------------------------------------------------------------ |
24
34
  | `FRUGAL_CUSTOMER_ID` | Customer ID issued by Frugal. When unset, the tenant label (auto-injected server-side by VMAuth from the bearer token) identifies the customer and `frugal.customer_id` is omitted. |
25
35
  | `FRUGAL_REPO_ROOT` | Absolute path to the source root; the strip prefix for `caller_file`. Defaults to the process working directory. |
26
- | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | Frugal-specific OTLP collector URL. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics`. Falls back to `OTEL_EXPORTER_OTLP_ENDPOINT`. |
27
- | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Overrides the header built from `FRUGAL_API_KEY`. Falls back to `OTEL_EXPORTER_OTLP_HEADERS`. |
36
+ | `FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT` | Frugal-specific OTLP collector URL. Defaults to `https://metrics.frugal.co/opentelemetry/v1/metrics` (also when set but empty). |
37
+ | `FRUGAL_OTEL_EXPORTER_OTLP_HEADERS` | Export auth headers (e.g. `Authorization=Bearer ...`). Precedence: this variable wins; otherwise the header built from `FRUGAL_API_KEY` (`FRUGAL_METRICS_API_KEY` first), which is sent verbatim. Values set here are percent-decoded (`Bearer%20tok` → `Bearer tok`). A value with no `key=value` pair is treated as missing auth (the API key is not used instead); a blank (whitespace-only) value is treated as unset. |
28
38
  | `FRUGAL_COMPONENTS` | Comma-separated logical components (e.g. `ai-handler,reco`). When set, each metric is ALSO emitted once per component (tagged `frugal.component`) on top of the base project series, so usage can be viewed for the whole project or split by component. The list is deduped (order-preserving) and only the first **10** entries are honored (hard cap, not configurable); the rest are dropped at startup with one debug warning. See [Cardinality caps](#cardinality-caps). |
29
- | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`; a value outside the first 20 entries (hard cap, not configurable) is coerced to `other`. |
39
+ | `FRUGAL_CUSTOM_LABEL` | Comma-separated allowlist of values for the `frugal.custom_label` label (e.g. `billing,search`). Applies only to `track_ai_call` / `start_ai_call` calls that pass `custom_label=...`. The vocabulary is the first **20** comma-separated entries by raw position (hard cap, not configurable); empty entries are ignored and duplicates collapsed. `other` is reserved (the coercion sentinel) and is ignored with one warning. A value outside the allowlist is coerced to `other` with a bounded once-per-value warning; when the variable is unset, every value is `other`. One warning fires only when non-empty entries past the 20th are actually dropped (a trailing comma does not warn). |
40
+ | `FRUGAL_METRICS_EXPORTER` | `otlp` (default) or `console` (print metrics to stdout; console mode needs only `FRUGAL_PROJECT_ID`; the API key, endpoint and header variables are ignored). Case-insensitive; any other value warns once and uses `otlp`. |
41
+ | `FRUGAL_METRICS_PROJECT_ID` / `FRUGAL_METRICS_API_KEY` | Namespaced overrides read before `FRUGAL_PROJECT_ID` / `FRUGAL_API_KEY`, for apps that already use those names. An empty value falls through to the bare name. |
30
42
  | `FRUGAL_PATH_BLOCK_LIST` | Comma-separated repo-relative paths to skip when attributing the caller (see below). |
31
- | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence. Default `60000`. |
43
+ | `FRUGAL_METRIC_EXPORT_INTERVAL_MS` | Export cadence in ms, `1`-`2147483647`. Default `60000`; a value outside the range warns once and uses the default. |
32
44
  | `FRUGAL_MAX_CALL_SITES` | Cardinality cap — max distinct `(caller_file, caller_function)` pairs to attribute before overflow. Default `500`, set to `0` to disable. See [Cardinality caps](#cardinality-caps). |
33
45
  | `FRUGAL_MAX_MODELS` | Cardinality cap — max distinct model strings to attribute before `gen_ai.request.model` / `gen_ai.response.model` overflow to `<overflow>`. One registry shared by both labels, so a request alias and its dated response snapshot each take a slot. Default `100`, set to `0` to disable. See [Cardinality caps](#cardinality-caps). |
34
46
  | `FRUGAL_MAX_SERIES_PER_INSTRUMENT` | OpenTelemetry's per-instrument limit on distinct attribute **combinations**, raised from the SDK default. Default `50000`, set to `0` for effectively unlimited (a 1,000,000 ceiling). See [Cardinality caps](#cardinality-caps). |
35
- | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls (0..1) where the lightweight prompt analyzer runs and emits derived metrics (`noise_ratio`, `cache_prefix_stable`, `few_shot_*`, …). Default `0.1`. Set to `0` to disable, `1` for every call. See [Prompt analysis](#prompt-analysis). |
47
+ | `FRUGAL_PROMPT_ANALYSIS_SAMPLE_RATE` | Fraction of calls where the lightweight prompt analyzer runs and emits derived metrics (`noise_ratio`, `cache_prefix_stable`, `few_shot_*`, …): a decimal in [0, 1], plain or scientific notation (e.g. `0.25`, `1e-1`); anything else warns once and uses `0.1`. Default `0.1`. Set to `0` to disable, `1` for every call. See [Prompt analysis](#prompt-analysis). |
36
48
  | `FRUGAL_DUPLICATE_REQUEST_DETECTION` | Duplicate-request detection (`frugal.gen_ai.duplicate_requests`) is **on by default**: every chat request is canonicalized and SHA-256 hashed in-process to classify it `first_seen`/`repeat_seen`. Set to `0`/`false`/`no`/`off` to turn it off entirely (no hashing, no filter memory, no metric). |
37
- | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `604800` (7 days). |
49
+ | `FRUGAL_DUPLICATE_WINDOW_SECONDS` | How long a request signature is remembered for duplicate detection. Default `3600`, max `86400` (1 day). |
38
50
  | `FRUGAL_DUPLICATE_CAPACITY` | Expected distinct request signatures per window — sizes the in-memory duplicate filter. Default `10000`, max `1000000`. Raise it if the debug callback warns the filter is saturated. If the filter's memory can't be allocated at all, detection disables itself for the rest of the process (one-time `error` diagnostic + one `internal_errors` point). |
39
51
  | `FRUGAL_DUPLICATE_FP_RATE` | Target false-positive rate of the duplicate filter. Default `0.01`, clamped to `[1e-6, 0.5]`. |
40
52
  | `FRUGAL_DUPLICATE_GENERATIONS` | Rotation granularity of the duplicate filter's time window. Default `2`, max `16`. |
41
53
  | `FRUGAL_DISABLED` | Kill switch. Set to `1` to disable without uninstalling. |
42
54
 
43
- Standard `OTEL_EXPORTER_OTLP_*` env vars are honored as fallbacks if the
44
- `FRUGAL_*` variants are unset.
45
-
46
55
  ## Wrap your clients
47
56
 
48
57
  **OpenAI**
@@ -296,7 +305,8 @@ extra calls are ignored.
296
305
  **Metadata** — `track_ai_call` / `start_ai_call` kwargs (all optional):
297
306
  `system` (→ `gen_ai.system`, default `"custom"`), `operation` (default
298
307
  `"chat"`), `request_model` (default `"<unknown>"`), `custom_label` (→ the
299
- `frugal.custom_label` label on every metric of the call; values outside the
308
+ `frugal.custom_label` label on every metric of the call except the
309
+ `frugal.gen_ai.internal_errors` setup_failed / emission points; values outside the
300
310
  `FRUGAL_CUSTOM_LABEL` allowlist become `other`). To also run the sampled
301
311
  prompt analyzer, pass any of `messages` / `system_prompt` / `tools` /
302
312
  `response_format` / `max_tokens`.
@@ -514,16 +524,27 @@ your handler).
514
524
 
515
525
  ## Verifying the setup
516
526
 
517
- Point the library at a local collector (or set the endpoint empty to print
518
- metrics to the console on the export interval):
527
+ Set `FRUGAL_METRICS_EXPORTER=console` to print the metrics to stdout instead
528
+ of exporting them. Console mode needs only `FRUGAL_PROJECT_ID`; the API key,
529
+ endpoint and header variables are ignored. An empty
530
+ endpoint does not select the console exporter: it falls through to the default
531
+ Frugal endpoint.
532
+
533
+ ```bash
534
+ export FRUGAL_PROJECT_ID=my-project
535
+ export FRUGAL_METRICS_EXPORTER=console
536
+ export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
537
+ python my_script.py
538
+ ```
539
+
540
+ Or point the library at a local OTLP/HTTP collector to see what it exports:
519
541
 
520
542
  ```bash
521
- export FRUGAL_API_KEY=local
543
+ export FRUGAL_API_KEY=test-key
522
544
  export FRUGAL_PROJECT_ID=my-project
523
- export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT= # empty → ConsoleMetricExporter
545
+ export FRUGAL_OTEL_EXPORTER_OTLP_ENDPOINT=http://localhost:4318/v1/metrics
524
546
  export FRUGAL_METRIC_EXPORT_INTERVAL_MS=5000
525
547
  python my_script.py
526
548
  ```
527
549
 
528
- Every five seconds you should see JSON metric records emitted with
529
- `gen_ai.*` and `frugal.*` attributes.
550
+ The collector should receive metric records with `gen_ai.*` and `frugal.*` attributes on each 5-second export that follows at least one wrapped call. Counters and histograms are exported as delta, so an idle interval carries no counter or histogram points: make a call, then watch the next export.
@@ -7,6 +7,6 @@
7
7
  "start": "node src/server.js"
8
8
  },
9
9
  "dependencies": {
10
- "express": "^4.19.2"
10
+ "express": "^5.2.1"
11
11
  }
12
12
  }