lutflow 0.3.2__tar.gz → 0.3.3__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lutflow-0.3.2 → lutflow-0.3.3}/CHANGELOG.md +92 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/PKG-INFO +52 -23
- {lutflow-0.3.2 → lutflow-0.3.3}/README.md +51 -22
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/__init__.py +8 -2
- lutflow-0.3.3/src/lutflow/budget.py +282 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/cli.py +33 -17
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/client.py +106 -9
- lutflow-0.3.3/src/lutflow/pricing.py +172 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/measurable.py +6 -6
- lutflow-0.3.3/src/lutflow/wrappers/_guard.py +90 -0
- lutflow-0.3.3/src/lutflow/wrappers/anthropic.py +182 -0
- lutflow-0.3.3/src/lutflow/wrappers/gemini.py +166 -0
- lutflow-0.3.3/src/lutflow/wrappers/openai.py +210 -0
- lutflow-0.3.2/src/lutflow/budget.py +0 -158
- lutflow-0.3.2/src/lutflow/pricing.py +0 -205
- lutflow-0.3.2/src/lutflow/wrappers/anthropic.py +0 -338
- lutflow-0.3.2/src/lutflow/wrappers/gemini.py +0 -254
- lutflow-0.3.2/src/lutflow/wrappers/openai.py +0 -278
- {lutflow-0.3.2 → lutflow-0.3.3}/.gitignore +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/LICENSE +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/pyproject.toml +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/executors/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/executors/self_kill.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/frontier.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/py.typed +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/adapters/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/adapters/base.py +8 -8
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/adapters/vllm.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/contract.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/economics.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/observe.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/probe.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/probe_prompts.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/prometheus.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/rates.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/render.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/tier0/report.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.3}/src/lutflow/wrappers/__init__.py +0 -0
|
@@ -5,6 +5,98 @@ All notable changes to the Lutflow SDK will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.3.3] - 2026-10-20
|
|
9
|
+
|
|
10
|
+
Integrity of the in-process wrappers, plus CLI help and `probe` fixes from the 0.3.2
|
|
11
|
+
audit. `tier0.measure.v1`, `tier0.check.v1` and the measurement paths of
|
|
12
|
+
`check`, `measure` and `probe` are unchanged.
|
|
13
|
+
|
|
14
|
+
### Behavior changes
|
|
15
|
+
|
|
16
|
+
Read this before upgrading from 0.3.2:
|
|
17
|
+
|
|
18
|
+
- **No built-in prices.** An unrecognized model was priced as `gpt-4o-mini`, and an
|
|
19
|
+
unrecognized GPU as an L4 at USD 0.80/h, from an undated table. Now a wrapped call to
|
|
20
|
+
a model without a declared price raises `PriceUnavailableError` **before** the
|
|
21
|
+
provider is called. `report_tokens`, `report_gpu_seconds` and `predict_from_usage`
|
|
22
|
+
raise it too. Declare prices with `Client(token_prices={model_id: TokenPrice(...)})`:
|
|
23
|
+
exact model ids, separate input and output USD per million tokens, a `source` and an
|
|
24
|
+
`effective_at` date. A GPU cost needs `gpu_hourly_rate_usd`.
|
|
25
|
+
- **The call that crosses the budget keeps its response.** In 0.3.2, `RAISE_ERROR`
|
|
26
|
+
raised after the provider answered, so the caller lost a response that had already
|
|
27
|
+
been paid for. Now the crossing call returns, its spend is recorded, and the **next**
|
|
28
|
+
wrapped call raises `BudgetExceededError` before reaching the provider. `WARN_ONLY`
|
|
29
|
+
keeps calling. `CALLBACK` is notified once; an exception it raises is logged and does
|
|
30
|
+
not replace the response. Spend already made is not reverted: this is not a billing
|
|
31
|
+
ceiling.
|
|
32
|
+
- **`SELF_KILL` no longer terminates the process from a wrapped call.** It refuses the
|
|
33
|
+
next call instead, and `Client` warns when it is selected. Manual `add_cost()` still
|
|
34
|
+
applies it.
|
|
35
|
+
- **Missing usage is not zero.** A response without the provider's usage, or with only
|
|
36
|
+
part of it, was estimated with `tiktoken` or counted as zero. Now it is returned, the
|
|
37
|
+
call is recorded as unmetered (`Client.unmetered_calls`), and later wrapped calls
|
|
38
|
+
raise `SpendUnverifiableError`, a `BudgetExceededError`. `WARN_ONLY` keeps calling.
|
|
39
|
+
Anthropic responses with cache tokens are unmetered as well: cache reads and writes
|
|
40
|
+
bill at their own rates.
|
|
41
|
+
- **A provider call that raises is unmetered too.** A timeout, a dropped connection or
|
|
42
|
+
an error response after the request may have been sent leaves its spend unknown: the
|
|
43
|
+
original exception propagates, the call counts in `Client.unmetered_calls`, and later
|
|
44
|
+
wrapped calls raise `SpendUnverifiableError` (`WARN_ONLY` keeps calling). A
|
|
45
|
+
connection error is included because the provider SDK retries on its own, and an
|
|
46
|
+
earlier attempt may have reached the provider. The wrapper's own refusals (no
|
|
47
|
+
`model`, no declared price, `stream=True`, budget) happen before the provider is
|
|
48
|
+
called and are not counted.
|
|
49
|
+
- **What is not metered is refused.** Metered methods: `chat.completions.create` and
|
|
50
|
+
`embeddings.create` (OpenAI), `messages.create` (Anthropic), and `generate_content`
|
|
51
|
+
and `start_chat().send_message` (Gemini), synchronous and without `stream=True`
|
|
52
|
+
(`lutflow.SUPPORTED_METHODS`). Streaming, async clients (`AsyncOpenAI`,
|
|
53
|
+
`AsyncAnthropic`) and every other resource or method (for example `responses`,
|
|
54
|
+
`completions`, `images`, `beta`, `messages.stream`) raise `UnsupportedMethodError`
|
|
55
|
+
without calling the provider. In 0.3.2 they were delegated and counted as zero.
|
|
56
|
+
Plain attributes pass through; the original client is `wrapped.unwrapped`.
|
|
57
|
+
- **`stop_gpu_timer()` without `start_gpu_timer()` raises `RuntimeError`** instead of
|
|
58
|
+
returning a cost of 0.0.
|
|
59
|
+
|
|
60
|
+
### Added
|
|
61
|
+
|
|
62
|
+
- `TokenPrice`, `PriceUnavailableError`, `SpendUnverifiableError`,
|
|
63
|
+
`UnsupportedMethodError` and `SUPPORTED_METHODS`, exported from `lutflow`.
|
|
64
|
+
- `Client(token_prices=...)` and `Client.unmetered_calls`.
|
|
65
|
+
- Gemini thinking models: `thoughts_token_count` counts as output tokens, as it is
|
|
66
|
+
billed.
|
|
67
|
+
|
|
68
|
+
### Changed
|
|
69
|
+
|
|
70
|
+
- **CLI help.** `check --help` says "measurable" instead of "Ready-medible" and states
|
|
71
|
+
the rule in plain terms, without an internal source path or a document the package
|
|
72
|
+
does not ship; its PASS/FAIL line says "measurable (vLLM v0.30.0 profile)".
|
|
73
|
+
`measure --help` no longer says "cost & energy" for the live path, where energy is
|
|
74
|
+
`unavailable`.
|
|
75
|
+
|
|
76
|
+
### Fixed
|
|
77
|
+
|
|
78
|
+
- **`probe` refuses a path-bearing `--endpoint` before its confirmation.** Its
|
|
79
|
+
requests go to `<endpoint>/v1/chat/completions`, so a path (a `kubectl proxy`
|
|
80
|
+
route, `.../metrics`) cannot be honoured. 0.3.2 asked for "yes", sent a POST under
|
|
81
|
+
the path and then exited 2. Now it exits 2 before the prompt, with no request, and
|
|
82
|
+
says to use `kubectl port-forward` or a client inside the cluster.
|
|
83
|
+
|
|
84
|
+
### Deprecated
|
|
85
|
+
|
|
86
|
+
- `price_per_1k_usd`: one rate for input and output. It still applies to models
|
|
87
|
+
without a `token_prices` entry and emits a `DeprecationWarning`.
|
|
88
|
+
|
|
89
|
+
### Removed
|
|
90
|
+
|
|
91
|
+
- `DEFAULT_MODEL_PRICING`, `DEFAULT_GPU_PRICING`, `GEMINI_PRICING`, `ModelPricing`
|
|
92
|
+
and `GPUPricing`, and the model-name normalization that mapped, for example,
|
|
93
|
+
`claude-sonnet-4-5` to Claude 3 Sonnet and `gemini-2.5-*` to Gemini 1.5.
|
|
94
|
+
- The wrappers' `acreate` methods, which called an `acreate` the provider clients do
|
|
95
|
+
not define (checked on openai 1.109.1 and 3.35.1 and anthropic 1.15.0), the
|
|
96
|
+
estimating Anthropic `messages.stream` wrapper and `WrappedStream`.
|
|
97
|
+
- The wrappers no longer call `tiktoken`, so a wrapped call does not download its
|
|
98
|
+
encoding file. The `count_tokens*` helpers remain and still may.
|
|
99
|
+
|
|
8
100
|
## [0.3.2] - 2026-10-17
|
|
9
101
|
|
|
10
102
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lutflow
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.3
|
|
4
4
|
Summary: Local cost intelligence for LLM inference: measure vLLM economics and enforce in-process budgets.
|
|
5
5
|
Project-URL: Homepage, https://lutflow.dev
|
|
6
6
|
Project-URL: Documentation, https://pypi.org/project/lutflow/
|
|
@@ -164,8 +164,8 @@ Requires Python 3.9+ and [uv](https://docs.astral.sh/uv/). Replace the URL with
|
|
|
164
164
|
**engine's** base URL, not a router. The server must already expose `/metrics`.
|
|
165
165
|
|
|
166
166
|
```bash
|
|
167
|
-
uvx --from 'lutflow>=0.3.
|
|
168
|
-
uvx --from 'lutflow>=0.3.
|
|
167
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow check --endpoint http://localhost:8000 &&
|
|
168
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow measure --endpoint http://localhost:8000 --duration 60 --interval 5 --json measurement.json &&
|
|
169
169
|
python3 -m json.tool measurement.json
|
|
170
170
|
```
|
|
171
171
|
|
|
@@ -202,7 +202,7 @@ typed metric families and counter creation markers. `check` judges one snapshot;
|
|
|
202
202
|
it does not run a benchmark or certify GPU readiness or model quality.
|
|
203
203
|
|
|
204
204
|
```bash
|
|
205
|
-
uvx --from 'lutflow>=0.3.
|
|
205
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow check --endpoint http://localhost:8000 --json check.json
|
|
206
206
|
```
|
|
207
207
|
|
|
208
208
|
Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
@@ -217,7 +217,7 @@ For an **instance** quote, set `USD_PER_HOUR`, `GPUS_IN_INSTANCE` and `GPU_COUNT
|
|
|
217
217
|
from your price source and deployment record; keep the quote's date and source.
|
|
218
218
|
|
|
219
219
|
```bash
|
|
220
|
-
uvx --from 'lutflow>=0.3.
|
|
220
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow measure --endpoint http://localhost:8000 \
|
|
221
221
|
--duration 60 --interval 5 --json measurement.json \
|
|
222
222
|
--gpu-hourly-rate "$USD_PER_HOUR" --rate-basis instance \
|
|
223
223
|
--gpus-in-basis "$GPUS_IN_INSTANCE" --gpu-count "$GPU_COUNT"
|
|
@@ -238,7 +238,7 @@ export offline and prints a table by concurrency: read-only, no network, no live
|
|
|
238
238
|
### `probe`: generate explicitly confirmed load
|
|
239
239
|
|
|
240
240
|
```bash
|
|
241
|
-
uvx --from 'lutflow>=0.3.
|
|
241
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow probe --endpoint http://localhost:8000 \
|
|
242
242
|
--model "$MODEL_ID" --concurrency 4 --duration 60 --json probe.json
|
|
243
243
|
```
|
|
244
244
|
|
|
@@ -251,26 +251,54 @@ proxy URLs are not supported for `probe`. Reports mark the load as induced.
|
|
|
251
251
|
|
|
252
252
|
Wrap an OpenAI, Anthropic or Gemini client with `Client.wrap(provider_client)`
|
|
253
253
|
to track spend and enforce a local budget. Wrapped calls still go to your provider.
|
|
254
|
-
Install `lutflow[openai]>=0.3.
|
|
255
|
-
|
|
254
|
+
Install `lutflow[openai]>=0.3.3,<0.4` and set `OPENAI_API_KEY`; the call uses your key.
|
|
255
|
+
lutflow ships no price table: declare the price you pay for each exact model id,
|
|
256
|
+
where you read it and the date it applied. The figures below are placeholders.
|
|
256
257
|
|
|
257
258
|
```python
|
|
258
259
|
import openai
|
|
259
|
-
from lutflow import BudgetStrategy, Client
|
|
260
|
-
|
|
261
|
-
client = Client(
|
|
260
|
+
from lutflow import BudgetStrategy, Client, TokenPrice
|
|
261
|
+
|
|
262
|
+
client = Client(
|
|
263
|
+
tenant_id="my-app",
|
|
264
|
+
budget_usd=10.0, # an example limit you choose
|
|
265
|
+
on_budget_exceeded=BudgetStrategy.RAISE_ERROR,
|
|
266
|
+
token_prices={
|
|
267
|
+
"gpt-4o-mini": TokenPrice(
|
|
268
|
+
input_usd_per_million=0.15, # replace with the price you pay
|
|
269
|
+
output_usd_per_million=0.60,
|
|
270
|
+
source="<the price page you read>",
|
|
271
|
+
effective_at="2026-10-01",
|
|
272
|
+
),
|
|
273
|
+
},
|
|
274
|
+
)
|
|
262
275
|
wrapped = client.wrap(openai.OpenAI())
|
|
263
276
|
wrapped.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "Hi!"}])
|
|
264
277
|
print(client.accumulated_cost_usd)
|
|
265
278
|
print(client.remaining_budget_usd)
|
|
266
279
|
```
|
|
267
280
|
|
|
268
|
-
The
|
|
269
|
-
|
|
270
|
-
|
|
281
|
+
The budget is checked before each wrapped call. The call that crosses it returns
|
|
282
|
+
its response and its spend is recorded; the next wrapped call raises
|
|
283
|
+
`BudgetExceededError` before it reaches the provider (`WARN_ONLY` lets it through,
|
|
284
|
+
`CALLBACK` is notified). Spend already made is not reverted: this is a local guard,
|
|
285
|
+
not a cap on your provider account, and it does not stop cluster workloads.
|
|
286
|
+
`SELF_KILL` no longer terminates the process from a wrapped call.
|
|
287
|
+
|
|
288
|
+
Metered: `chat.completions.create` and `embeddings.create` (OpenAI),
|
|
289
|
+
`messages.create` (Anthropic), `generate_content` and chat `send_message`
|
|
290
|
+
(Gemini), synchronous and without `stream=True`, priced from the usage the
|
|
291
|
+
provider returns. Other methods, streaming and async clients raise
|
|
292
|
+
`UnsupportedMethodError` without calling the provider. A model without a declared
|
|
293
|
+
price raises `PriceUnavailableError`. A response without usage is returned, and
|
|
294
|
+
later wrapped calls raise `SpendUnverifiableError`; so do the calls after a provider
|
|
295
|
+
call that raised, because that request may have run.
|
|
271
296
|
|
|
272
297
|
## What's new in 0.3.x
|
|
273
298
|
|
|
299
|
+
- **0.3.3:** wrapper integrity: no built-in prices or substituted rates, the
|
|
300
|
+
response that crosses the budget is kept and later calls stop, and methods the
|
|
301
|
+
wrappers do not meter are refused.
|
|
274
302
|
- **0.3.2:** window-mean TPOT from recent vLLM inter-token histograms; a warning
|
|
275
303
|
identifies the histogram bucket used to interpolate p95.
|
|
276
304
|
- **0.3.1:** `check` evaluates telemetry structure before a measurement window.
|
|
@@ -287,10 +315,11 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
287
315
|
- **Sampling limits resolution.** Productive duty cycle reflects scrape intervals.
|
|
288
316
|
Histogram p95 is interpolated inside a bucket, not an exact tail measurement.
|
|
289
317
|
TPOT is a per-token-gap mean, not a per-request mean.
|
|
290
|
-
- **Wrapper
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
318
|
+
- **Wrapper costs come from your prices.** A cost is the provider-reported tokens
|
|
319
|
+
times the price you declare; it is not reconciled with an invoice. OpenAI and
|
|
320
|
+
Gemini cached input tokens are priced at the full input rate; Anthropic cache
|
|
321
|
+
tokens leave the call unmetered. The deprecated `price_per_1k_usd` applies one
|
|
322
|
+
rate to input and output.
|
|
294
323
|
- **Endpoint access uses your environment.** Proxy and certificate variables apply.
|
|
295
324
|
JSON from `check` includes the endpoint path, which can identify a namespace or
|
|
296
325
|
pod; review it before sharing. No report is uploaded to Lutflow.
|
|
@@ -300,18 +329,18 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
300
329
|
For a persistent CLI:
|
|
301
330
|
|
|
302
331
|
```bash
|
|
303
|
-
uv tool install 'lutflow>=0.3.
|
|
332
|
+
uv tool install 'lutflow>=0.3.3,<0.4'
|
|
304
333
|
```
|
|
305
334
|
|
|
306
335
|
For Python code, install inside your virtual environment:
|
|
307
336
|
|
|
308
337
|
```bash
|
|
309
|
-
python -m pip install 'lutflow>=0.3.
|
|
338
|
+
python -m pip install 'lutflow>=0.3.3,<0.4'
|
|
310
339
|
```
|
|
311
340
|
|
|
312
|
-
Provider extras: `'lutflow[openai]>=0.3.
|
|
313
|
-
`'lutflow[anthropic]>=0.3.
|
|
314
|
-
or `'lutflow[all]>=0.3.
|
|
341
|
+
Provider extras: `'lutflow[openai]>=0.3.3,<0.4'`,
|
|
342
|
+
`'lutflow[anthropic]>=0.3.3,<0.4'`, `'lutflow[gemini]>=0.3.3,<0.4'`,
|
|
343
|
+
or `'lutflow[all]>=0.3.3,<0.4'`. They add provider SDKs; the vLLM CLI needs no extra.
|
|
315
344
|
The `anthropic` and `all` extras require Python 3.10+; the CLI and other extras support Python 3.9+.
|
|
316
345
|
|
|
317
346
|
## License
|
|
@@ -12,8 +12,8 @@ Requires Python 3.9+ and [uv](https://docs.astral.sh/uv/). Replace the URL with
|
|
|
12
12
|
**engine's** base URL, not a router. The server must already expose `/metrics`.
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
|
-
uvx --from 'lutflow>=0.3.
|
|
16
|
-
uvx --from 'lutflow>=0.3.
|
|
15
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow check --endpoint http://localhost:8000 &&
|
|
16
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow measure --endpoint http://localhost:8000 --duration 60 --interval 5 --json measurement.json &&
|
|
17
17
|
python3 -m json.tool measurement.json
|
|
18
18
|
```
|
|
19
19
|
|
|
@@ -50,7 +50,7 @@ typed metric families and counter creation markers. `check` judges one snapshot;
|
|
|
50
50
|
it does not run a benchmark or certify GPU readiness or model quality.
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
uvx --from 'lutflow>=0.3.
|
|
53
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow check --endpoint http://localhost:8000 --json check.json
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
@@ -65,7 +65,7 @@ For an **instance** quote, set `USD_PER_HOUR`, `GPUS_IN_INSTANCE` and `GPU_COUNT
|
|
|
65
65
|
from your price source and deployment record; keep the quote's date and source.
|
|
66
66
|
|
|
67
67
|
```bash
|
|
68
|
-
uvx --from 'lutflow>=0.3.
|
|
68
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow measure --endpoint http://localhost:8000 \
|
|
69
69
|
--duration 60 --interval 5 --json measurement.json \
|
|
70
70
|
--gpu-hourly-rate "$USD_PER_HOUR" --rate-basis instance \
|
|
71
71
|
--gpus-in-basis "$GPUS_IN_INSTANCE" --gpu-count "$GPU_COUNT"
|
|
@@ -86,7 +86,7 @@ export offline and prints a table by concurrency: read-only, no network, no live
|
|
|
86
86
|
### `probe`: generate explicitly confirmed load
|
|
87
87
|
|
|
88
88
|
```bash
|
|
89
|
-
uvx --from 'lutflow>=0.3.
|
|
89
|
+
uvx --from 'lutflow>=0.3.3,<0.4' lutflow probe --endpoint http://localhost:8000 \
|
|
90
90
|
--model "$MODEL_ID" --concurrency 4 --duration 60 --json probe.json
|
|
91
91
|
```
|
|
92
92
|
|
|
@@ -99,26 +99,54 @@ proxy URLs are not supported for `probe`. Reports mark the load as induced.
|
|
|
99
99
|
|
|
100
100
|
Wrap an OpenAI, Anthropic or Gemini client with `Client.wrap(provider_client)`
|
|
101
101
|
to track spend and enforce a local budget. Wrapped calls still go to your provider.
|
|
102
|
-
Install `lutflow[openai]>=0.3.
|
|
103
|
-
|
|
102
|
+
Install `lutflow[openai]>=0.3.3,<0.4` and set `OPENAI_API_KEY`; the call uses your key.
|
|
103
|
+
lutflow ships no price table: declare the price you pay for each exact model id,
|
|
104
|
+
where you read it and the date it applied. The figures below are placeholders.
|
|
104
105
|
|
|
105
106
|
```python
|
|
106
107
|
import openai
|
|
107
|
-
from lutflow import BudgetStrategy, Client
|
|
108
|
-
|
|
109
|
-
client = Client(
|
|
108
|
+
from lutflow import BudgetStrategy, Client, TokenPrice
|
|
109
|
+
|
|
110
|
+
client = Client(
|
|
111
|
+
tenant_id="my-app",
|
|
112
|
+
budget_usd=10.0, # an example limit you choose
|
|
113
|
+
on_budget_exceeded=BudgetStrategy.RAISE_ERROR,
|
|
114
|
+
token_prices={
|
|
115
|
+
"gpt-4o-mini": TokenPrice(
|
|
116
|
+
input_usd_per_million=0.15, # replace with the price you pay
|
|
117
|
+
output_usd_per_million=0.60,
|
|
118
|
+
source="<the price page you read>",
|
|
119
|
+
effective_at="2026-10-01",
|
|
120
|
+
),
|
|
121
|
+
},
|
|
122
|
+
)
|
|
110
123
|
wrapped = client.wrap(openai.OpenAI())
|
|
111
124
|
wrapped.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "Hi!"}])
|
|
112
125
|
print(client.accumulated_cost_usd)
|
|
113
126
|
print(client.remaining_budget_usd)
|
|
114
127
|
```
|
|
115
128
|
|
|
116
|
-
The
|
|
117
|
-
|
|
118
|
-
|
|
129
|
+
The budget is checked before each wrapped call. The call that crosses it returns
|
|
130
|
+
its response and its spend is recorded; the next wrapped call raises
|
|
131
|
+
`BudgetExceededError` before it reaches the provider (`WARN_ONLY` lets it through,
|
|
132
|
+
`CALLBACK` is notified). Spend already made is not reverted: this is a local guard,
|
|
133
|
+
not a cap on your provider account, and it does not stop cluster workloads.
|
|
134
|
+
`SELF_KILL` no longer terminates the process from a wrapped call.
|
|
135
|
+
|
|
136
|
+
Metered: `chat.completions.create` and `embeddings.create` (OpenAI),
|
|
137
|
+
`messages.create` (Anthropic), `generate_content` and chat `send_message`
|
|
138
|
+
(Gemini), synchronous and without `stream=True`, priced from the usage the
|
|
139
|
+
provider returns. Other methods, streaming and async clients raise
|
|
140
|
+
`UnsupportedMethodError` without calling the provider. A model without a declared
|
|
141
|
+
price raises `PriceUnavailableError`. A response without usage is returned, and
|
|
142
|
+
later wrapped calls raise `SpendUnverifiableError`; so do the calls after a provider
|
|
143
|
+
call that raised, because that request may have run.
|
|
119
144
|
|
|
120
145
|
## What's new in 0.3.x
|
|
121
146
|
|
|
147
|
+
- **0.3.3:** wrapper integrity: no built-in prices or substituted rates, the
|
|
148
|
+
response that crosses the budget is kept and later calls stop, and methods the
|
|
149
|
+
wrappers do not meter are refused.
|
|
122
150
|
- **0.3.2:** window-mean TPOT from recent vLLM inter-token histograms; a warning
|
|
123
151
|
identifies the histogram bucket used to interpolate p95.
|
|
124
152
|
- **0.3.1:** `check` evaluates telemetry structure before a measurement window.
|
|
@@ -135,10 +163,11 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
135
163
|
- **Sampling limits resolution.** Productive duty cycle reflects scrape intervals.
|
|
136
164
|
Histogram p95 is interpolated inside a bucket, not an exact tail measurement.
|
|
137
165
|
TPOT is a per-token-gap mean, not a per-request mean.
|
|
138
|
-
- **Wrapper
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
166
|
+
- **Wrapper costs come from your prices.** A cost is the provider-reported tokens
|
|
167
|
+
times the price you declare; it is not reconciled with an invoice. OpenAI and
|
|
168
|
+
Gemini cached input tokens are priced at the full input rate; Anthropic cache
|
|
169
|
+
tokens leave the call unmetered. The deprecated `price_per_1k_usd` applies one
|
|
170
|
+
rate to input and output.
|
|
142
171
|
- **Endpoint access uses your environment.** Proxy and certificate variables apply.
|
|
143
172
|
JSON from `check` includes the endpoint path, which can identify a namespace or
|
|
144
173
|
pod; review it before sharing. No report is uploaded to Lutflow.
|
|
@@ -148,18 +177,18 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
148
177
|
For a persistent CLI:
|
|
149
178
|
|
|
150
179
|
```bash
|
|
151
|
-
uv tool install 'lutflow>=0.3.
|
|
180
|
+
uv tool install 'lutflow>=0.3.3,<0.4'
|
|
152
181
|
```
|
|
153
182
|
|
|
154
183
|
For Python code, install inside your virtual environment:
|
|
155
184
|
|
|
156
185
|
```bash
|
|
157
|
-
python -m pip install 'lutflow>=0.3.
|
|
186
|
+
python -m pip install 'lutflow>=0.3.3,<0.4'
|
|
158
187
|
```
|
|
159
188
|
|
|
160
|
-
Provider extras: `'lutflow[openai]>=0.3.
|
|
161
|
-
`'lutflow[anthropic]>=0.3.
|
|
162
|
-
or `'lutflow[all]>=0.3.
|
|
189
|
+
Provider extras: `'lutflow[openai]>=0.3.3,<0.4'`,
|
|
190
|
+
`'lutflow[anthropic]>=0.3.3,<0.4'`, `'lutflow[gemini]>=0.3.3,<0.4'`,
|
|
191
|
+
or `'lutflow[all]>=0.3.3,<0.4'`. They add provider SDKs; the vLLM CLI needs no extra.
|
|
163
192
|
The `anthropic` and `all` extras require Python 3.10+; the CLI and other extras support Python 3.9+.
|
|
164
193
|
|
|
165
194
|
## License
|
|
@@ -20,9 +20,10 @@ Example with budget_session context manager:
|
|
|
20
20
|
from importlib.metadata import PackageNotFoundError
|
|
21
21
|
from importlib.metadata import version as _pkg_version
|
|
22
22
|
|
|
23
|
-
from lutflow.budget import BudgetExceededError, BudgetStrategy
|
|
23
|
+
from lutflow.budget import BudgetExceededError, BudgetStrategy, SpendUnverifiableError
|
|
24
24
|
from lutflow.client import Client, budget_session
|
|
25
|
-
from lutflow.pricing import PricingMode
|
|
25
|
+
from lutflow.pricing import PriceUnavailableError, PricingMode, TokenPrice
|
|
26
|
+
from lutflow.wrappers._guard import SUPPORTED_METHODS, UnsupportedMethodError
|
|
26
27
|
|
|
27
28
|
try:
|
|
28
29
|
# Single source of truth: the installed package metadata, which hatch-vcs
|
|
@@ -35,5 +36,10 @@ __all__ = [
|
|
|
35
36
|
"budget_session",
|
|
36
37
|
"BudgetStrategy",
|
|
37
38
|
"BudgetExceededError",
|
|
39
|
+
"SpendUnverifiableError",
|
|
38
40
|
"PricingMode",
|
|
41
|
+
"TokenPrice",
|
|
42
|
+
"PriceUnavailableError",
|
|
43
|
+
"UnsupportedMethodError",
|
|
44
|
+
"SUPPORTED_METHODS",
|
|
39
45
|
]
|