lutflow 0.3.2__tar.gz → 0.3.4__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {lutflow-0.3.2 → lutflow-0.3.4}/CHANGELOG.md +128 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/PKG-INFO +67 -23
- {lutflow-0.3.2 → lutflow-0.3.4}/README.md +66 -22
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/__init__.py +8 -2
- lutflow-0.3.4/src/lutflow/budget.py +282 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/cli.py +200 -18
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/client.py +106 -9
- lutflow-0.3.4/src/lutflow/pricing.py +172 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/adapters/base.py +19 -17
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/adapters/vllm.py +2 -0
- lutflow-0.3.4/src/lutflow/tier0/live.py +744 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/measurable.py +6 -6
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/probe.py +7 -1
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/prometheus.py +28 -1
- lutflow-0.3.4/src/lutflow/wrappers/_guard.py +90 -0
- lutflow-0.3.4/src/lutflow/wrappers/anthropic.py +182 -0
- lutflow-0.3.4/src/lutflow/wrappers/gemini.py +166 -0
- lutflow-0.3.4/src/lutflow/wrappers/openai.py +210 -0
- lutflow-0.3.2/src/lutflow/budget.py +0 -158
- lutflow-0.3.2/src/lutflow/pricing.py +0 -205
- lutflow-0.3.2/src/lutflow/wrappers/anthropic.py +0 -338
- lutflow-0.3.2/src/lutflow/wrappers/gemini.py +0 -254
- lutflow-0.3.2/src/lutflow/wrappers/openai.py +0 -278
- {lutflow-0.3.2 → lutflow-0.3.4}/.gitignore +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/LICENSE +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/pyproject.toml +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/executors/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/executors/self_kill.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/frontier.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/py.typed +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/adapters/__init__.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/contract.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/economics.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/observe.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/probe_prompts.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/rates.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/render.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/tier0/report.py +0 -0
- {lutflow-0.3.2 → lutflow-0.3.4}/src/lutflow/wrappers/__init__.py +0 -0
|
@@ -5,6 +5,134 @@ All notable changes to the Lutflow SDK will be documented in this file.
|
|
|
5
5
|
The format is based on [Keep a Changelog](https://keepachangelog.com/en/1.0.0/),
|
|
6
6
|
and this project adheres to [Semantic Versioning](https://semver.org/spec/v2.0.0.html).
|
|
7
7
|
|
|
8
|
+
## [0.3.4] - 2026-11-07
|
|
9
|
+
|
|
10
|
+
`lutflow top`, a read-only live view of one vLLM engine. `tier0.measure.v1` and
|
|
11
|
+
`tier0.check.v1` are unchanged. The measurement paths of `check`, `measure` and `probe`
|
|
12
|
+
are unchanged except for how the metrics URL is built and how a loopback endpoint is
|
|
13
|
+
reached (see Fixed).
|
|
14
|
+
|
|
15
|
+
0.3.3 reached PyPI on 2026-10-28 at 13:24 UTC. The wheel and the sdist that PyPI serves
|
|
16
|
+
have the sha256 of its reviewed G3 build (run 40681041842). Its section below keeps the
|
|
17
|
+
date its source was frozen, 2026-10-20.
|
|
18
|
+
|
|
19
|
+
### Added
|
|
20
|
+
|
|
21
|
+
- **`lutflow top`**: a read-only terminal view of one vLLM engine over a
|
|
22
|
+
rolling window (tokens, latency, running and waiting requests, allocated cost with a
|
|
23
|
+
declared rate), with `--replay` on SYNTHETIC data. It starts no load and writes
|
|
24
|
+
nothing. It reads no GPU sensor: power, energy and utilization are unavailable.
|
|
25
|
+
Exit 130 on Ctrl-C, 2 usage or no interactive terminal, 1 an unexpected error.
|
|
26
|
+
- `EngineSample.requests_waiting`: the engine's queue at the scrape
|
|
27
|
+
(`vllm:num_requests_waiting`, filtered by model like `requests_running`). `None`
|
|
28
|
+
when the engine does not publish it; absence is not an empty queue. It is the last
|
|
29
|
+
field, so a positional `EngineSample(...)` keeps working. `tier0.measure.v1` does not
|
|
30
|
+
carry it.
|
|
31
|
+
|
|
32
|
+
### Fixed
|
|
33
|
+
|
|
34
|
+
- **A query string on `--endpoint` stays after the metrics path.** 0.3.3 and earlier turned
|
|
35
|
+
`http://host:8000/?token=x` into `http://host:8000/?token=x/metrics`, and cut a
|
|
36
|
+
trailing slash off the query's last value. Now it is
|
|
37
|
+
`http://host:8000/metrics?token=x`.
|
|
38
|
+
- **A loopback endpoint is scraped and probed without proxies.** `127.0.0.0/8`, `::1`
|
|
39
|
+
and `localhost` (a `kubectl port-forward`) were sent to `HTTP_PROXY` or the macOS
|
|
40
|
+
system proxy when one was set, with any credential in the URL; `probe`'s synthetic
|
|
41
|
+
load too. Now they are reached directly; other endpoints keep the environment's
|
|
42
|
+
proxy settings.
|
|
43
|
+
|
|
44
|
+
## [0.3.3] - 2026-10-20
|
|
45
|
+
|
|
46
|
+
Integrity of the in-process wrappers, plus CLI help and `probe` fixes from the 0.3.2
|
|
47
|
+
audit. `tier0.measure.v1`, `tier0.check.v1` and the measurement paths of
|
|
48
|
+
`check`, `measure` and `probe` are unchanged.
|
|
49
|
+
|
|
50
|
+
### Behavior changes
|
|
51
|
+
|
|
52
|
+
Read this before upgrading from 0.3.2:
|
|
53
|
+
|
|
54
|
+
- **No built-in prices.** An unrecognized model was priced as `gpt-4o-mini`, and an
|
|
55
|
+
unrecognized GPU as an L4 at USD 0.80/h, from an undated table. Now a wrapped call to
|
|
56
|
+
a model without a declared price raises `PriceUnavailableError` **before** the
|
|
57
|
+
provider is called. `report_tokens`, `report_gpu_seconds` and `predict_from_usage`
|
|
58
|
+
raise it too. Declare prices with `Client(token_prices={model_id: TokenPrice(...)})`:
|
|
59
|
+
exact model ids, separate input and output USD per million tokens, a `source` and an
|
|
60
|
+
`effective_at` date. A GPU cost needs `gpu_hourly_rate_usd`.
|
|
61
|
+
- **The call that crosses the budget keeps its response.** In 0.3.2, `RAISE_ERROR`
|
|
62
|
+
raised after the provider answered, so the caller lost a response that had already
|
|
63
|
+
been paid for. Now the crossing call returns, its spend is recorded, and the **next**
|
|
64
|
+
wrapped call raises `BudgetExceededError` before reaching the provider. `WARN_ONLY`
|
|
65
|
+
keeps calling. `CALLBACK` is notified once; an exception it raises is logged and does
|
|
66
|
+
not replace the response. Spend already made is not reverted: this is not a billing
|
|
67
|
+
ceiling.
|
|
68
|
+
- **`SELF_KILL` no longer terminates the process from a wrapped call.** It refuses the
|
|
69
|
+
next call instead, and `Client` warns when it is selected. Manual `add_cost()` still
|
|
70
|
+
applies it.
|
|
71
|
+
- **Missing usage is not zero.** A response without the provider's usage, or with only
|
|
72
|
+
part of it, was estimated with `tiktoken` or counted as zero. Now it is returned, the
|
|
73
|
+
call is recorded as unmetered (`Client.unmetered_calls`), and later wrapped calls
|
|
74
|
+
raise `SpendUnverifiableError`, a `BudgetExceededError`. `WARN_ONLY` keeps calling.
|
|
75
|
+
Anthropic responses with cache tokens are unmetered as well: cache reads and writes
|
|
76
|
+
bill at their own rates.
|
|
77
|
+
- **A provider call that raises is unmetered too.** A timeout, a dropped connection or
|
|
78
|
+
an error response after the request may have been sent leaves its spend unknown: the
|
|
79
|
+
original exception propagates, the call counts in `Client.unmetered_calls`, and later
|
|
80
|
+
wrapped calls raise `SpendUnverifiableError` (`WARN_ONLY` keeps calling). A
|
|
81
|
+
connection error is included because the provider SDK retries on its own, and an
|
|
82
|
+
earlier attempt may have reached the provider. The wrapper's own refusals (no
|
|
83
|
+
`model`, no declared price, `stream=True`, budget) happen before the provider is
|
|
84
|
+
called and are not counted.
|
|
85
|
+
- **What is not metered is refused.** Metered methods: `chat.completions.create` and
|
|
86
|
+
`embeddings.create` (OpenAI), `messages.create` (Anthropic), and `generate_content`
|
|
87
|
+
and `start_chat().send_message` (Gemini), synchronous and without `stream=True`
|
|
88
|
+
(`lutflow.SUPPORTED_METHODS`). Streaming, async clients (`AsyncOpenAI`,
|
|
89
|
+
`AsyncAnthropic`) and every other resource or method (for example `responses`,
|
|
90
|
+
`completions`, `images`, `beta`, `messages.stream`) raise `UnsupportedMethodError`
|
|
91
|
+
without calling the provider. In 0.3.2 they were delegated and counted as zero.
|
|
92
|
+
Plain attributes pass through; the original client is `wrapped.unwrapped`.
|
|
93
|
+
- **`stop_gpu_timer()` without `start_gpu_timer()` raises `RuntimeError`** instead of
|
|
94
|
+
returning a cost of 0.0.
|
|
95
|
+
|
|
96
|
+
### Added
|
|
97
|
+
|
|
98
|
+
- `TokenPrice`, `PriceUnavailableError`, `SpendUnverifiableError`,
|
|
99
|
+
`UnsupportedMethodError` and `SUPPORTED_METHODS`, exported from `lutflow`.
|
|
100
|
+
- `Client(token_prices=...)` and `Client.unmetered_calls`.
|
|
101
|
+
- Gemini thinking models: `thoughts_token_count` counts as output tokens, as it is
|
|
102
|
+
billed.
|
|
103
|
+
|
|
104
|
+
### Changed
|
|
105
|
+
|
|
106
|
+
- **CLI help.** `check --help` says "measurable" instead of "Ready-medible" and states
|
|
107
|
+
the rule in plain terms, without an internal source path or a document the package
|
|
108
|
+
does not ship; its PASS/FAIL line says "measurable (vLLM v0.30.0 profile)".
|
|
109
|
+
`measure --help` no longer says "cost & energy" for the live path, where energy is
|
|
110
|
+
`unavailable`.
|
|
111
|
+
|
|
112
|
+
### Fixed
|
|
113
|
+
|
|
114
|
+
- **`probe` refuses a path-bearing `--endpoint` before its confirmation.** Its
|
|
115
|
+
requests go to `<endpoint>/v1/chat/completions`, so a path (a `kubectl proxy`
|
|
116
|
+
route, `.../metrics`) cannot be honoured. 0.3.2 asked for "yes", sent a POST under
|
|
117
|
+
the path and then exited 2. Now it exits 2 before the prompt, with no request, and
|
|
118
|
+
says to use `kubectl port-forward` or a client inside the cluster.
|
|
119
|
+
|
|
120
|
+
### Deprecated
|
|
121
|
+
|
|
122
|
+
- `price_per_1k_usd`: one rate for input and output. It still applies to models
|
|
123
|
+
without a `token_prices` entry and emits a `DeprecationWarning`.
|
|
124
|
+
|
|
125
|
+
### Removed
|
|
126
|
+
|
|
127
|
+
- `DEFAULT_MODEL_PRICING`, `DEFAULT_GPU_PRICING`, `GEMINI_PRICING`, `ModelPricing`
|
|
128
|
+
and `GPUPricing`, and the model-name normalization that mapped, for example,
|
|
129
|
+
`claude-sonnet-4-5` to Claude 3 Sonnet and `gemini-2.5-*` to Gemini 1.5.
|
|
130
|
+
- The wrappers' `acreate` methods, which called an `acreate` the provider clients do
|
|
131
|
+
not define (checked on openai 1.109.1 and 3.35.1 and anthropic 1.15.0), the
|
|
132
|
+
estimating Anthropic `messages.stream` wrapper and `WrappedStream`.
|
|
133
|
+
- The wrappers no longer call `tiktoken`, so a wrapped call does not download its
|
|
134
|
+
encoding file. The `count_tokens*` helpers remain and still may.
|
|
135
|
+
|
|
8
136
|
## [0.3.2] - 2026-10-17
|
|
9
137
|
|
|
10
138
|
### Changed
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: lutflow
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.4
|
|
4
4
|
Summary: Local cost intelligence for LLM inference: measure vLLM economics and enforce in-process budgets.
|
|
5
5
|
Project-URL: Homepage, https://lutflow.dev
|
|
6
6
|
Project-URL: Documentation, https://pypi.org/project/lutflow/
|
|
@@ -164,8 +164,8 @@ Requires Python 3.9+ and [uv](https://docs.astral.sh/uv/). Replace the URL with
|
|
|
164
164
|
**engine's** base URL, not a router. The server must already expose `/metrics`.
|
|
165
165
|
|
|
166
166
|
```bash
|
|
167
|
-
uvx --from 'lutflow>=0.3.
|
|
168
|
-
uvx --from 'lutflow>=0.3.
|
|
167
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow check --endpoint http://localhost:8000 &&
|
|
168
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow measure --endpoint http://localhost:8000 --duration 60 --interval 5 --json measurement.json &&
|
|
169
169
|
python3 -m json.tool measurement.json
|
|
170
170
|
```
|
|
171
171
|
|
|
@@ -202,7 +202,7 @@ typed metric families and counter creation markers. `check` judges one snapshot;
|
|
|
202
202
|
it does not run a benchmark or certify GPU readiness or model quality.
|
|
203
203
|
|
|
204
204
|
```bash
|
|
205
|
-
uvx --from 'lutflow>=0.3.
|
|
205
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow check --endpoint http://localhost:8000 --json check.json
|
|
206
206
|
```
|
|
207
207
|
|
|
208
208
|
Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
@@ -210,6 +210,18 @@ Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
|
210
210
|
Add `--model "$MODEL_ID"` to require that model's metrics and `/v1/models` entry.
|
|
211
211
|
Use `--metrics-file saved.prom` instead of `--endpoint` to check a saved snapshot.
|
|
212
212
|
|
|
213
|
+
### `top`: watch an engine live
|
|
214
|
+
|
|
215
|
+
```bash
|
|
216
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow top --endpoint http://localhost:8000
|
|
217
|
+
```
|
|
218
|
+
|
|
219
|
+
A read-only terminal view of one vLLM engine: tokens, latency, running and waiting
|
|
220
|
+
requests and, with a declared `--gpu-hourly-rate`, allocated cost over the last
|
|
221
|
+
`--window` seconds. It sends nothing, reads no prompts and writes no file. It reads
|
|
222
|
+
no GPU sensor: power, energy and utilization are unavailable. `lutflow top --replay`
|
|
223
|
+
shows the view on SYNTHETIC data. It needs an interactive terminal.
|
|
224
|
+
|
|
213
225
|
### `measure`: observe existing traffic
|
|
214
226
|
|
|
215
227
|
To include monetary fields, supply a sourced quote and the allocation it covers.
|
|
@@ -217,7 +229,7 @@ For an **instance** quote, set `USD_PER_HOUR`, `GPUS_IN_INSTANCE` and `GPU_COUNT
|
|
|
217
229
|
from your price source and deployment record; keep the quote's date and source.
|
|
218
230
|
|
|
219
231
|
```bash
|
|
220
|
-
uvx --from 'lutflow>=0.3.
|
|
232
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow measure --endpoint http://localhost:8000 \
|
|
221
233
|
--duration 60 --interval 5 --json measurement.json \
|
|
222
234
|
--gpu-hourly-rate "$USD_PER_HOUR" --rate-basis instance \
|
|
223
235
|
--gpus-in-basis "$GPUS_IN_INSTANCE" --gpu-count "$GPU_COUNT"
|
|
@@ -238,7 +250,7 @@ export offline and prints a table by concurrency: read-only, no network, no live
|
|
|
238
250
|
### `probe`: generate explicitly confirmed load
|
|
239
251
|
|
|
240
252
|
```bash
|
|
241
|
-
uvx --from 'lutflow>=0.3.
|
|
253
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow probe --endpoint http://localhost:8000 \
|
|
242
254
|
--model "$MODEL_ID" --concurrency 4 --duration 60 --json probe.json
|
|
243
255
|
```
|
|
244
256
|
|
|
@@ -251,26 +263,57 @@ proxy URLs are not supported for `probe`. Reports mark the load as induced.
|
|
|
251
263
|
|
|
252
264
|
Wrap an OpenAI, Anthropic or Gemini client with `Client.wrap(provider_client)`
|
|
253
265
|
to track spend and enforce a local budget. Wrapped calls still go to your provider.
|
|
254
|
-
Install `lutflow[openai]>=0.3.
|
|
255
|
-
|
|
266
|
+
Install `lutflow[openai]>=0.3.4,<0.4` and set `OPENAI_API_KEY`; the call uses your key.
|
|
267
|
+
lutflow ships no price table: declare the price you pay for each exact model id,
|
|
268
|
+
where you read it and the date it applied. The figures below are placeholders.
|
|
256
269
|
|
|
257
270
|
```python
|
|
258
271
|
import openai
|
|
259
|
-
from lutflow import BudgetStrategy, Client
|
|
260
|
-
|
|
261
|
-
client = Client(
|
|
272
|
+
from lutflow import BudgetStrategy, Client, TokenPrice
|
|
273
|
+
|
|
274
|
+
client = Client(
|
|
275
|
+
tenant_id="my-app",
|
|
276
|
+
budget_usd=10.0, # an example limit you choose
|
|
277
|
+
on_budget_exceeded=BudgetStrategy.RAISE_ERROR,
|
|
278
|
+
token_prices={
|
|
279
|
+
"gpt-4o-mini": TokenPrice(
|
|
280
|
+
input_usd_per_million=0.15, # replace with the price you pay
|
|
281
|
+
output_usd_per_million=0.60,
|
|
282
|
+
source="<the price page you read>",
|
|
283
|
+
effective_at="2026-10-01",
|
|
284
|
+
),
|
|
285
|
+
},
|
|
286
|
+
)
|
|
262
287
|
wrapped = client.wrap(openai.OpenAI())
|
|
263
288
|
wrapped.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "Hi!"}])
|
|
264
289
|
print(client.accumulated_cost_usd)
|
|
265
290
|
print(client.remaining_budget_usd)
|
|
266
291
|
```
|
|
267
292
|
|
|
268
|
-
The
|
|
269
|
-
|
|
270
|
-
|
|
293
|
+
The budget is checked before each wrapped call. The call that crosses it returns
|
|
294
|
+
its response and its spend is recorded; the next wrapped call raises
|
|
295
|
+
`BudgetExceededError` before it reaches the provider (`WARN_ONLY` lets it through,
|
|
296
|
+
`CALLBACK` is notified). Spend already made is not reverted: this is a local guard,
|
|
297
|
+
not a cap on your provider account, and it does not stop cluster workloads.
|
|
298
|
+
`SELF_KILL` no longer terminates the process from a wrapped call.
|
|
299
|
+
|
|
300
|
+
Metered: `chat.completions.create` and `embeddings.create` (OpenAI),
|
|
301
|
+
`messages.create` (Anthropic), `generate_content` and chat `send_message`
|
|
302
|
+
(Gemini), synchronous and without `stream=True`, priced from the usage the
|
|
303
|
+
provider returns. Other methods, streaming and async clients raise
|
|
304
|
+
`UnsupportedMethodError` without calling the provider. A model without a declared
|
|
305
|
+
price raises `PriceUnavailableError`. A response without usage is returned, and
|
|
306
|
+
later wrapped calls raise `SpendUnverifiableError`; so do the calls after a provider
|
|
307
|
+
call that raised, because that request may have run.
|
|
271
308
|
|
|
272
309
|
## What's new in 0.3.x
|
|
273
310
|
|
|
311
|
+
- **0.3.4:** `lutflow top`, a read-only live view of one vLLM engine; a query string
|
|
312
|
+
on `--endpoint` stays after the metrics path, and a loopback endpoint is reached
|
|
313
|
+
without proxies.
|
|
314
|
+
- **0.3.3:** wrapper integrity: no built-in prices or substituted rates, the
|
|
315
|
+
response that crosses the budget is kept and later calls stop, and methods the
|
|
316
|
+
wrappers do not meter are refused.
|
|
274
317
|
- **0.3.2:** window-mean TPOT from recent vLLM inter-token histograms; a warning
|
|
275
318
|
identifies the histogram bucket used to interpolate p95.
|
|
276
319
|
- **0.3.1:** `check` evaluates telemetry structure before a measurement window.
|
|
@@ -287,10 +330,11 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
287
330
|
- **Sampling limits resolution.** Productive duty cycle reflects scrape intervals.
|
|
288
331
|
Histogram p95 is interpolated inside a bucket, not an exact tail measurement.
|
|
289
332
|
TPOT is a per-token-gap mean, not a per-request mean.
|
|
290
|
-
- **Wrapper
|
|
291
|
-
|
|
292
|
-
|
|
293
|
-
|
|
333
|
+
- **Wrapper costs come from your prices.** A cost is the provider-reported tokens
|
|
334
|
+
times the price you declare; it is not reconciled with an invoice. OpenAI and
|
|
335
|
+
Gemini cached input tokens are priced at the full input rate; Anthropic cache
|
|
336
|
+
tokens leave the call unmetered. The deprecated `price_per_1k_usd` applies one
|
|
337
|
+
rate to input and output.
|
|
294
338
|
- **Endpoint access uses your environment.** Proxy and certificate variables apply.
|
|
295
339
|
JSON from `check` includes the endpoint path, which can identify a namespace or
|
|
296
340
|
pod; review it before sharing. No report is uploaded to Lutflow.
|
|
@@ -300,18 +344,18 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
300
344
|
For a persistent CLI:
|
|
301
345
|
|
|
302
346
|
```bash
|
|
303
|
-
uv tool install 'lutflow>=0.3.
|
|
347
|
+
uv tool install 'lutflow>=0.3.4,<0.4'
|
|
304
348
|
```
|
|
305
349
|
|
|
306
350
|
For Python code, install inside your virtual environment:
|
|
307
351
|
|
|
308
352
|
```bash
|
|
309
|
-
python -m pip install 'lutflow>=0.3.
|
|
353
|
+
python -m pip install 'lutflow>=0.3.4,<0.4'
|
|
310
354
|
```
|
|
311
355
|
|
|
312
|
-
Provider extras: `'lutflow[openai]>=0.3.
|
|
313
|
-
`'lutflow[anthropic]>=0.3.
|
|
314
|
-
or `'lutflow[all]>=0.3.
|
|
356
|
+
Provider extras: `'lutflow[openai]>=0.3.4,<0.4'`,
|
|
357
|
+
`'lutflow[anthropic]>=0.3.4,<0.4'`, `'lutflow[gemini]>=0.3.4,<0.4'`,
|
|
358
|
+
or `'lutflow[all]>=0.3.4,<0.4'`. They add provider SDKs; the vLLM CLI needs no extra.
|
|
315
359
|
The `anthropic` and `all` extras require Python 3.10+; the CLI and other extras support Python 3.9+.
|
|
316
360
|
|
|
317
361
|
## License
|
|
@@ -12,8 +12,8 @@ Requires Python 3.9+ and [uv](https://docs.astral.sh/uv/). Replace the URL with
|
|
|
12
12
|
**engine's** base URL, not a router. The server must already expose `/metrics`.
|
|
13
13
|
|
|
14
14
|
```bash
|
|
15
|
-
uvx --from 'lutflow>=0.3.
|
|
16
|
-
uvx --from 'lutflow>=0.3.
|
|
15
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow check --endpoint http://localhost:8000 &&
|
|
16
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow measure --endpoint http://localhost:8000 --duration 60 --interval 5 --json measurement.json &&
|
|
17
17
|
python3 -m json.tool measurement.json
|
|
18
18
|
```
|
|
19
19
|
|
|
@@ -50,7 +50,7 @@ typed metric families and counter creation markers. `check` judges one snapshot;
|
|
|
50
50
|
it does not run a benchmark or certify GPU readiness or model quality.
|
|
51
51
|
|
|
52
52
|
```bash
|
|
53
|
-
uvx --from 'lutflow>=0.3.
|
|
53
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow check --endpoint http://localhost:8000 --json check.json
|
|
54
54
|
```
|
|
55
55
|
|
|
56
56
|
Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
@@ -58,6 +58,18 @@ Exit codes: **0** pass, **1** demonstrated failure, **3** unreadable/no verdict,
|
|
|
58
58
|
Add `--model "$MODEL_ID"` to require that model's metrics and `/v1/models` entry.
|
|
59
59
|
Use `--metrics-file saved.prom` instead of `--endpoint` to check a saved snapshot.
|
|
60
60
|
|
|
61
|
+
### `top`: watch an engine live
|
|
62
|
+
|
|
63
|
+
```bash
|
|
64
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow top --endpoint http://localhost:8000
|
|
65
|
+
```
|
|
66
|
+
|
|
67
|
+
A read-only terminal view of one vLLM engine: tokens, latency, running and waiting
|
|
68
|
+
requests and, with a declared `--gpu-hourly-rate`, allocated cost over the last
|
|
69
|
+
`--window` seconds. It sends nothing, reads no prompts and writes no file. It reads
|
|
70
|
+
no GPU sensor: power, energy and utilization are unavailable. `lutflow top --replay`
|
|
71
|
+
shows the view on SYNTHETIC data. It needs an interactive terminal.
|
|
72
|
+
|
|
61
73
|
### `measure`: observe existing traffic
|
|
62
74
|
|
|
63
75
|
To include monetary fields, supply a sourced quote and the allocation it covers.
|
|
@@ -65,7 +77,7 @@ For an **instance** quote, set `USD_PER_HOUR`, `GPUS_IN_INSTANCE` and `GPU_COUNT
|
|
|
65
77
|
from your price source and deployment record; keep the quote's date and source.
|
|
66
78
|
|
|
67
79
|
```bash
|
|
68
|
-
uvx --from 'lutflow>=0.3.
|
|
80
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow measure --endpoint http://localhost:8000 \
|
|
69
81
|
--duration 60 --interval 5 --json measurement.json \
|
|
70
82
|
--gpu-hourly-rate "$USD_PER_HOUR" --rate-basis instance \
|
|
71
83
|
--gpus-in-basis "$GPUS_IN_INSTANCE" --gpu-count "$GPU_COUNT"
|
|
@@ -86,7 +98,7 @@ export offline and prints a table by concurrency: read-only, no network, no live
|
|
|
86
98
|
### `probe`: generate explicitly confirmed load
|
|
87
99
|
|
|
88
100
|
```bash
|
|
89
|
-
uvx --from 'lutflow>=0.3.
|
|
101
|
+
uvx --from 'lutflow>=0.3.4,<0.4' lutflow probe --endpoint http://localhost:8000 \
|
|
90
102
|
--model "$MODEL_ID" --concurrency 4 --duration 60 --json probe.json
|
|
91
103
|
```
|
|
92
104
|
|
|
@@ -99,26 +111,57 @@ proxy URLs are not supported for `probe`. Reports mark the load as induced.
|
|
|
99
111
|
|
|
100
112
|
Wrap an OpenAI, Anthropic or Gemini client with `Client.wrap(provider_client)`
|
|
101
113
|
to track spend and enforce a local budget. Wrapped calls still go to your provider.
|
|
102
|
-
Install `lutflow[openai]>=0.3.
|
|
103
|
-
|
|
114
|
+
Install `lutflow[openai]>=0.3.4,<0.4` and set `OPENAI_API_KEY`; the call uses your key.
|
|
115
|
+
lutflow ships no price table: declare the price you pay for each exact model id,
|
|
116
|
+
where you read it and the date it applied. The figures below are placeholders.
|
|
104
117
|
|
|
105
118
|
```python
|
|
106
119
|
import openai
|
|
107
|
-
from lutflow import BudgetStrategy, Client
|
|
108
|
-
|
|
109
|
-
client = Client(
|
|
120
|
+
from lutflow import BudgetStrategy, Client, TokenPrice
|
|
121
|
+
|
|
122
|
+
client = Client(
|
|
123
|
+
tenant_id="my-app",
|
|
124
|
+
budget_usd=10.0, # an example limit you choose
|
|
125
|
+
on_budget_exceeded=BudgetStrategy.RAISE_ERROR,
|
|
126
|
+
token_prices={
|
|
127
|
+
"gpt-4o-mini": TokenPrice(
|
|
128
|
+
input_usd_per_million=0.15, # replace with the price you pay
|
|
129
|
+
output_usd_per_million=0.60,
|
|
130
|
+
source="<the price page you read>",
|
|
131
|
+
effective_at="2026-10-01",
|
|
132
|
+
),
|
|
133
|
+
},
|
|
134
|
+
)
|
|
110
135
|
wrapped = client.wrap(openai.OpenAI())
|
|
111
136
|
wrapped.chat.completions.create(model="gpt-4o-mini", messages=[{"role": "user", "content": "Hi!"}])
|
|
112
137
|
print(client.accumulated_cost_usd)
|
|
113
138
|
print(client.remaining_budget_usd)
|
|
114
139
|
```
|
|
115
140
|
|
|
116
|
-
The
|
|
117
|
-
|
|
118
|
-
|
|
141
|
+
The budget is checked before each wrapped call. The call that crosses it returns
|
|
142
|
+
its response and its spend is recorded; the next wrapped call raises
|
|
143
|
+
`BudgetExceededError` before it reaches the provider (`WARN_ONLY` lets it through,
|
|
144
|
+
`CALLBACK` is notified). Spend already made is not reverted: this is a local guard,
|
|
145
|
+
not a cap on your provider account, and it does not stop cluster workloads.
|
|
146
|
+
`SELF_KILL` no longer terminates the process from a wrapped call.
|
|
147
|
+
|
|
148
|
+
Metered: `chat.completions.create` and `embeddings.create` (OpenAI),
|
|
149
|
+
`messages.create` (Anthropic), `generate_content` and chat `send_message`
|
|
150
|
+
(Gemini), synchronous and without `stream=True`, priced from the usage the
|
|
151
|
+
provider returns. Other methods, streaming and async clients raise
|
|
152
|
+
`UnsupportedMethodError` without calling the provider. A model without a declared
|
|
153
|
+
price raises `PriceUnavailableError`. A response without usage is returned, and
|
|
154
|
+
later wrapped calls raise `SpendUnverifiableError`; so do the calls after a provider
|
|
155
|
+
call that raised, because that request may have run.
|
|
119
156
|
|
|
120
157
|
## What's new in 0.3.x
|
|
121
158
|
|
|
159
|
+
- **0.3.4:** `lutflow top`, a read-only live view of one vLLM engine; a query string
|
|
160
|
+
on `--endpoint` stays after the metrics path, and a loopback endpoint is reached
|
|
161
|
+
without proxies.
|
|
162
|
+
- **0.3.3:** wrapper integrity: no built-in prices or substituted rates, the
|
|
163
|
+
response that crosses the budget is kept and later calls stop, and methods the
|
|
164
|
+
wrappers do not meter are refused.
|
|
122
165
|
- **0.3.2:** window-mean TPOT from recent vLLM inter-token histograms; a warning
|
|
123
166
|
identifies the histogram bucket used to interpolate p95.
|
|
124
167
|
- **0.3.1:** `check` evaluates telemetry structure before a measurement window.
|
|
@@ -135,10 +178,11 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
135
178
|
- **Sampling limits resolution.** Productive duty cycle reflects scrape intervals.
|
|
136
179
|
Histogram p95 is interpolated inside a bucket, not an exact tail measurement.
|
|
137
180
|
TPOT is a per-token-gap mean, not a per-request mean.
|
|
138
|
-
- **Wrapper
|
|
139
|
-
|
|
140
|
-
|
|
141
|
-
|
|
181
|
+
- **Wrapper costs come from your prices.** A cost is the provider-reported tokens
|
|
182
|
+
times the price you declare; it is not reconciled with an invoice. OpenAI and
|
|
183
|
+
Gemini cached input tokens are priced at the full input rate; Anthropic cache
|
|
184
|
+
tokens leave the call unmetered. The deprecated `price_per_1k_usd` applies one
|
|
185
|
+
rate to input and output.
|
|
142
186
|
- **Endpoint access uses your environment.** Proxy and certificate variables apply.
|
|
143
187
|
JSON from `check` includes the endpoint path, which can identify a namespace or
|
|
144
188
|
pod; review it before sharing. No report is uploaded to Lutflow.
|
|
@@ -148,18 +192,18 @@ process; it does not stop cluster workloads or cap your provider account.
|
|
|
148
192
|
For a persistent CLI:
|
|
149
193
|
|
|
150
194
|
```bash
|
|
151
|
-
uv tool install 'lutflow>=0.3.
|
|
195
|
+
uv tool install 'lutflow>=0.3.4,<0.4'
|
|
152
196
|
```
|
|
153
197
|
|
|
154
198
|
For Python code, install inside your virtual environment:
|
|
155
199
|
|
|
156
200
|
```bash
|
|
157
|
-
python -m pip install 'lutflow>=0.3.
|
|
201
|
+
python -m pip install 'lutflow>=0.3.4,<0.4'
|
|
158
202
|
```
|
|
159
203
|
|
|
160
|
-
Provider extras: `'lutflow[openai]>=0.3.
|
|
161
|
-
`'lutflow[anthropic]>=0.3.
|
|
162
|
-
or `'lutflow[all]>=0.3.
|
|
204
|
+
Provider extras: `'lutflow[openai]>=0.3.4,<0.4'`,
|
|
205
|
+
`'lutflow[anthropic]>=0.3.4,<0.4'`, `'lutflow[gemini]>=0.3.4,<0.4'`,
|
|
206
|
+
or `'lutflow[all]>=0.3.4,<0.4'`. They add provider SDKs; the vLLM CLI needs no extra.
|
|
163
207
|
The `anthropic` and `all` extras require Python 3.10+; the CLI and other extras support Python 3.9+.
|
|
164
208
|
|
|
165
209
|
## License
|
|
@@ -20,9 +20,10 @@ Example with budget_session context manager:
|
|
|
20
20
|
from importlib.metadata import PackageNotFoundError
|
|
21
21
|
from importlib.metadata import version as _pkg_version
|
|
22
22
|
|
|
23
|
-
from lutflow.budget import BudgetExceededError, BudgetStrategy
|
|
23
|
+
from lutflow.budget import BudgetExceededError, BudgetStrategy, SpendUnverifiableError
|
|
24
24
|
from lutflow.client import Client, budget_session
|
|
25
|
-
from lutflow.pricing import PricingMode
|
|
25
|
+
from lutflow.pricing import PriceUnavailableError, PricingMode, TokenPrice
|
|
26
|
+
from lutflow.wrappers._guard import SUPPORTED_METHODS, UnsupportedMethodError
|
|
26
27
|
|
|
27
28
|
try:
|
|
28
29
|
# Single source of truth: the installed package metadata, which hatch-vcs
|
|
@@ -35,5 +36,10 @@ __all__ = [
|
|
|
35
36
|
"budget_session",
|
|
36
37
|
"BudgetStrategy",
|
|
37
38
|
"BudgetExceededError",
|
|
39
|
+
"SpendUnverifiableError",
|
|
38
40
|
"PricingMode",
|
|
41
|
+
"TokenPrice",
|
|
42
|
+
"PriceUnavailableError",
|
|
43
|
+
"UnsupportedMethodError",
|
|
44
|
+
"SUPPORTED_METHODS",
|
|
39
45
|
]
|