unalloc 0.3.0__tar.gz → 0.3.2__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- {unalloc-0.3.0 → unalloc-0.3.2}/CHANGELOG.md +21 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/PKG-INFO +34 -11
- {unalloc-0.3.0 → unalloc-0.3.2}/README.md +33 -10
- {unalloc-0.3.0 → unalloc-0.3.2}/pyproject.toml +1 -1
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/__init__.py +1 -1
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/cli.py +48 -1
- unalloc-0.3.2/src/unalloc/fixtures/README.md +16 -0
- unalloc-0.3.2/src/unalloc/fixtures/invoices.json +7 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/fixtures/litellm_spend.json +4 -4
- unalloc-0.3.2/tests/test_cli.py +71 -0
- unalloc-0.3.0/tests/test_cli.py +0 -33
- {unalloc-0.3.0 → unalloc-0.3.2}/.gitignore +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/LICENSE +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/__init__.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/attribute.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/models.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/normalize.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/provenance.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/core/reconcile.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/fixtures/anthropic_cost_report.json +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/fixtures/openai_costs.json +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/fixtures/opencost_allocation.json +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/render/__init__.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/render/table.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/__init__.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/anthropic.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/base.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/litellm.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/openai.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/src/unalloc/sources/opencost.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_attribute.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_docs_build.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_fetch.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_normalize.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_paper_lint.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_paper_numbers.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_provenance.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_reconcile.py +0 -0
- {unalloc-0.3.0 → unalloc-0.3.2}/tests/test_sources.py +0 -0
|
@@ -1,5 +1,26 @@
|
|
|
1
1
|
# Changelog
|
|
2
2
|
|
|
3
|
+
## 0.3.2 — 2026-10-05
|
|
4
|
+
|
|
5
|
+
### Added
|
|
6
|
+
- `unalloc --version` (and `-V`), alongside the existing `unalloc version` command.
|
|
7
|
+
- `reconcile --fixtures` reconciles against bundled sample invoices when no `--invoice` is given: Anthropic ties out within tolerance and OpenAI shows an $381.44 (8.6%) gap, so the invoice check can be demoed without inventing a number. An explicit `--invoice` replaces the bundled ones.
|
|
8
|
+
|
|
9
|
+
### Changed
|
|
10
|
+
- The bundled LiteLLM fixture now routes to Azure OpenAI, Vertex AI and Bedrock instead of the OpenAI and Anthropic APIs, so the default four-source demo no longer counts gateway spend that the direct provider fixtures could also bill. Dollar amounts are unchanged: the fixture headline is still 73.5% of $66,630.38. `src/unalloc/fixtures/README.md` documents what each fixture covers.
|
|
11
|
+
- The quick start shows `--fallback cost_center` and `reconcile --fixtures`; `make demo` uses the bundled invoices.
|
|
12
|
+
|
|
13
|
+
Fixture attribution results are unchanged. The LiteLLM fixture bytes changed, so its provenance digest differs from 0.3.1.
|
|
14
|
+
|
|
15
|
+
## 0.3.1 — 2026-10-05
|
|
16
|
+
|
|
17
|
+
### Documentation
|
|
18
|
+
- Prominent source-overlap warning: the CLI defaults to all four sources and appends normalized rows without automatic economic deduplication. Source selection must account for both coverage and overlap.
|
|
19
|
+
- Research summaries describe disagreement between allocation rules rather than established overcharging or mispricing. GPU queue claims are limited to no waiting observed at about 0.5-second telemetry sampling.
|
|
20
|
+
- Synthetic billing and illustrative allocation pools, and the one-GPU/model/run-per-load limits, are explicit.
|
|
21
|
+
|
|
22
|
+
Accounting behavior and source defaults are unchanged. This patch includes the corrected README in package metadata and distributions; provenance reports version 0.3.1.
|
|
23
|
+
|
|
3
24
|
## 0.3.0 — 2026-10-04
|
|
4
25
|
|
|
5
26
|
### Added
|
|
@@ -1,6 +1,6 @@
|
|
|
1
1
|
Metadata-Version: 2.5
|
|
2
2
|
Name: unalloc
|
|
3
|
-
Version: 0.3.
|
|
3
|
+
Version: 0.3.2
|
|
4
4
|
Summary: Find the AI spend nobody owns. Joins OpenCost allocation data with LLM provider bills and reports what's unattributed.
|
|
5
5
|
Project-URL: Homepage, https://github.com/timurista/unalloc
|
|
6
6
|
Project-URL: Issues, https://github.com/timurista/unalloc/issues
|
|
@@ -55,6 +55,8 @@ Description-Content-Type: text/markdown
|
|
|
55
55
|
|
|
56
56
|
OpenCost tells you what your Kubernetes workloads cost. Your provider dashboard tells you what your OpenAI and Anthropic calls cost. Getting both into one view is possible today — OpenCost has an OpenAI plugin — but the answer still depends on a join nobody checks: the same dollar can arrive through both your gateway and your provider bill, and the labels that say who owns it are set per workload, not per pod template. `unalloc` pulls all four sources into one normalized ledger, joins them on a label dimension you choose, and reports the number your finance team keeps asking for: how much of this month's AI spend can't be attributed to any team — and how much of the rest was only rescued by a fallback key.
|
|
57
57
|
|
|
58
|
+
> **Avoid double counting before interpreting totals.** By default, the CLI requests all four sources: OpenCost, LiteLLM, OpenAI and Anthropic. It appends their normalized `CostRow`s into one ledger; it does **not** automatically deduplicate the same economic spend across sources. Gateway charges can also appear in provider bills (or OpenCost plugin output). Choose non-overlapping sources for your setup. For traffic fully covered by LiteLLM, use `--source opencost --source litellm` only if OpenCost excludes those same provider charges. Direct-provider-only traffic needs a separate coverage plan. The bundled fixtures are built not to overlap: their LiteLLM rows route to Azure OpenAI, Vertex AI and Bedrock, which unalloc does not pull directly, and their OpenCost rows are self-hosted workloads only ([fixture notes](https://github.com/timurista/unalloc/blob/main/src/unalloc/fixtures/README.md)). Fixture totals are synthetic demonstrations, not validated unique spend.
|
|
59
|
+
|
|
58
60
|
## The output
|
|
59
61
|
|
|
60
62
|
```
|
|
@@ -86,11 +88,11 @@ $ unalloc labels --fixtures --dimension team
|
|
|
86
88
|
┡━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
87
89
|
│ 1 │ opencost │ inference/vllm-llama-70b │ $19,147.10 │ cluster, controller, ... │
|
|
88
90
|
│ 2 │ opencost │ __idle__ │ $6,070.00 │ cluster │
|
|
89
|
-
│ 3 │ litellm │ claude-opus-5
|
|
91
|
+
│ 3 │ litellm │ bedrock/claude-opus-5 │ $5,412.75 │ environment, model │
|
|
90
92
|
└────┴──────────┴───────────────────────────┴────────────┴────────────────────────────┘
|
|
91
93
|
```
|
|
92
94
|
|
|
93
|
-
`inference/vllm-llama-70b` is the whole pitch: $19K of GPU spend carrying a `costCenter` label but no `team`, sitting next to $5K of Claude
|
|
95
|
+
`inference/vllm-llama-70b` is the whole pitch: $19K of GPU spend carrying a `costCenter` label but no `team`, sitting next to $5K of Claude spend routed through the gateway to Bedrock on an unlabeled legacy key. Two systems, one gap, no existing tool that shows them in the same table.
|
|
94
96
|
|
|
95
97
|
Add a fallback and unalloc says how much of the "allocated" spend only got there through it, so a fallback onto something that is not an owner can't hide inside the headline:
|
|
96
98
|
|
|
@@ -101,11 +103,30 @@ $ unalloc report --fixtures -D team --fallback namespace
|
|
|
101
103
|
│ $23,745.60 attributed only via fallback (namespace) │
|
|
102
104
|
```
|
|
103
105
|
|
|
106
|
+
Then check the ledger against what the providers billed. With `--fixtures`, the bundled sample invoices are used:
|
|
107
|
+
|
|
108
|
+
```
|
|
109
|
+
$ unalloc reconcile --fixtures
|
|
110
|
+
┏━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
|
|
111
|
+
┃ Source ┃ Ledger ┃ Invoice ┃ Delta ┃ Status ┃
|
|
112
|
+
┡━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
|
|
113
|
+
│ anthropic │ $8,737.03 │ $8,779.84 │ $-42.81 (-0.5%) │ matched │
|
|
114
|
+
│ litellm │ $11,444.27 │ — │ — │ no-invoice │
|
|
115
|
+
│ openai │ $4,031.36 │ $4,412.80 │ $-381.44 (-8.6%) │ under │
|
|
116
|
+
│ opencost │ $42,417.72 │ — │ — │ no-invoice │
|
|
117
|
+
└───────────┴────────────┴───────────┴──────────────────┴────────────┘
|
|
118
|
+
The ledger does not tie out. Attribution built on this is not defensible yet.
|
|
119
|
+
```
|
|
120
|
+
|
|
121
|
+
Anthropic agrees within the 1% tolerance. OpenAI billed $381 more than its costs endpoint reported, so any attribution built on that ledger understates OpenAI spend until the gap is explained.
|
|
122
|
+
|
|
104
123
|
## Install
|
|
105
124
|
|
|
106
125
|
```bash
|
|
107
|
-
pip install unalloc
|
|
108
|
-
unalloc report --fixtures
|
|
126
|
+
pip install unalloc # or: uv tool install unalloc
|
|
127
|
+
unalloc report --fixtures --fallback cost_center # works immediately, no infrastructure needed
|
|
128
|
+
unalloc reconcile --fixtures # ledger vs. sample invoices
|
|
129
|
+
unalloc --version
|
|
109
130
|
```
|
|
110
131
|
|
|
111
132
|
From a clone:
|
|
@@ -119,11 +140,11 @@ make check # ruff + pytest, same as CI
|
|
|
119
140
|
Or as a container, which needs no Python on the host:
|
|
120
141
|
|
|
121
142
|
```bash
|
|
122
|
-
docker build -t unalloc
|
|
123
|
-
docker run --rm unalloc
|
|
143
|
+
docker build -t unalloc .
|
|
144
|
+
docker run --rm unalloc report --fixtures -D team
|
|
124
145
|
|
|
125
146
|
# against real systems, reachable from the container
|
|
126
|
-
docker run --rm --network host --env-file .env unalloc
|
|
147
|
+
docker run --rm --network host --env-file .env unalloc report -D team
|
|
127
148
|
```
|
|
128
149
|
|
|
129
150
|
Copy `.env.example` to `.env` for the source configuration. unalloc stores
|
|
@@ -165,7 +186,9 @@ Configure sources by environment variable:
|
|
|
165
186
|
| `ANTHROPIC_ADMIN_KEY` | Anthropic org admin key (cost report endpoint, sent as `x-api-key`) |
|
|
166
187
|
| `UNALLOC_OPENAI_URL`, `UNALLOC_ANTHROPIC_URL` | Optional overrides for the public API roots, e.g. a proxy |
|
|
167
188
|
|
|
168
|
-
Restrict the run with `--source` to avoid double counting: if
|
|
189
|
+
Restrict the run with `--source` to avoid double counting: if LiteLLM covers all provider traffic and OpenCost excludes overlapping provider charges, use `--source opencost --source litellm` and skip the direct provider adapters. Confirm coverage and overlap for your setup. The [end-to-end case study](#case-studies-and-paper) demonstrates the resulting double count with synthetic billing and invoices; it does not measure customer savings.
|
|
190
|
+
|
|
191
|
+
The case-study invoices and monthly cost pools are synthetic or illustrative. The GPU result covers one H100, one model and one two-minute run per load; waiting-request telemetry was sampled about every 0.5 seconds. Allocation-rule disagreement does not establish a uniquely correct charge or production savings.
|
|
169
192
|
|
|
170
193
|
## Reproducing a result
|
|
171
194
|
|
|
@@ -234,11 +257,11 @@ Six studies push real (or realistically simulated) inference workloads through u
|
|
|
234
257
|
| Study | What runs | Headline |
|
|
235
258
|
| --- | --- | --- |
|
|
236
259
|
| [`kv_cache`](https://github.com/timurista/unalloc/blob/main/case_studies/kv_cache) | Discrete-event vLLM-style engine: paged KV blocks, prefix caching, continuous batching, preemption | Step-time metering shows ~0% idle once any request is in flight; KV-memory metering leaves a large share of the bill unowned |
|
|
237
|
-
| [`torch_kv`](https://github.com/timurista/unalloc/blob/main/case_studies/torch_kv) | A from-scratch PyTorch decoder with a real KV cache serving a 4-tenant trace |
|
|
260
|
+
| [`torch_kv`](https://github.com/timurista/unalloc/blob/main/case_studies/torch_kv) | A from-scratch PyTorch decoder with a real KV cache serving a 4-tenant trace | Token and measured-compute allocation rules differ by 33 points for a RAG tenant ($5.7K of an illustrative monthly pool) |
|
|
238
261
|
| [`distributed`](https://github.com/timurista/unalloc/blob/main/case_studies/distributed) | Tensor and pipeline parallel on `torch.distributed` (gloo), verified against a single-process reference | Owner label on LeaderWorkerSet leaders only: 66% unallocated; falling back to `name` "fixes" it by sending 61% to a Helm chart name |
|
|
239
262
|
| [`hybrid_e2e`](https://github.com/timurista/unalloc/blob/main/case_studies/hybrid_e2e) | The real CLI as a subprocess against live mock OpenCost / LiteLLM / OpenAI / Anthropic APIs (Postgres-backed LiteLLM in the dev container) | Enabling every source double counts $11.8K; reading only page one of the billing APIs reports 25% of spend |
|
|
240
263
|
| [`use_cases`](https://github.com/timurista/unalloc/blob/main/case_studies/use_cases) | Labeling Pareto, per-feature unit economics, self-host break-even, CI budget gate | Three label fixes take a 67%-unallocated org under 10% |
|
|
241
|
-
| [`gpu_validation`](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation) | Real vLLM 0.29 serving Qwen2.5-7B on a DigitalOcean H100, driven with the simulator's tenants ([runbook](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation/RUNBOOK.md), [evidence](https://github.com/timurista/unalloc/tree/main/case_studies/results/gpu_validation/evidence)) | Token
|
|
264
|
+
| [`gpu_validation`](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation) | Real vLLM 0.29 serving Qwen2.5-7B on a DigitalOcean H100, driven with the simulator's tenants ([runbook](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation/RUNBOOK.md), [evidence](https://github.com/timurista/unalloc/tree/main/case_studies/results/gpu_validation/evidence)) | Token and equal time-share allocation rules differ by 12–14 points for search; GPU "utilization" reads 97% at 2 req/s and 99% at 16 req/s |
|
|
242
265
|
|
|
243
266
|
```bash
|
|
244
267
|
make research # build ./.venv-research: CPU torch, notebook tooling, typst
|
|
@@ -13,6 +13,8 @@
|
|
|
13
13
|
|
|
14
14
|
OpenCost tells you what your Kubernetes workloads cost. Your provider dashboard tells you what your OpenAI and Anthropic calls cost. Getting both into one view is possible today — OpenCost has an OpenAI plugin — but the answer still depends on a join nobody checks: the same dollar can arrive through both your gateway and your provider bill, and the labels that say who owns it are set per workload, not per pod template. `unalloc` pulls all four sources into one normalized ledger, joins them on a label dimension you choose, and reports the number your finance team keeps asking for: how much of this month's AI spend can't be attributed to any team — and how much of the rest was only rescued by a fallback key.
|
|
15
15
|
|
|
16
|
+
> **Avoid double counting before interpreting totals.** By default, the CLI requests all four sources: OpenCost, LiteLLM, OpenAI and Anthropic. It appends their normalized `CostRow`s into one ledger; it does **not** automatically deduplicate the same economic spend across sources. Gateway charges can also appear in provider bills (or OpenCost plugin output). Choose non-overlapping sources for your setup. For traffic fully covered by LiteLLM, use `--source opencost --source litellm` only if OpenCost excludes those same provider charges. Direct-provider-only traffic needs a separate coverage plan. The bundled fixtures are built not to overlap: their LiteLLM rows route to Azure OpenAI, Vertex AI and Bedrock, which unalloc does not pull directly, and their OpenCost rows are self-hosted workloads only ([fixture notes](https://github.com/timurista/unalloc/blob/main/src/unalloc/fixtures/README.md)). Fixture totals are synthetic demonstrations, not validated unique spend.
|
|
17
|
+
|
|
16
18
|
## The output
|
|
17
19
|
|
|
18
20
|
```
|
|
@@ -44,11 +46,11 @@ $ unalloc labels --fixtures --dimension team
|
|
|
44
46
|
┡━━━━╇━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━━━━━━━━━━━┩
|
|
45
47
|
│ 1 │ opencost │ inference/vllm-llama-70b │ $19,147.10 │ cluster, controller, ... │
|
|
46
48
|
│ 2 │ opencost │ __idle__ │ $6,070.00 │ cluster │
|
|
47
|
-
│ 3 │ litellm │ claude-opus-5
|
|
49
|
+
│ 3 │ litellm │ bedrock/claude-opus-5 │ $5,412.75 │ environment, model │
|
|
48
50
|
└────┴──────────┴───────────────────────────┴────────────┴────────────────────────────┘
|
|
49
51
|
```
|
|
50
52
|
|
|
51
|
-
`inference/vllm-llama-70b` is the whole pitch: $19K of GPU spend carrying a `costCenter` label but no `team`, sitting next to $5K of Claude
|
|
53
|
+
`inference/vllm-llama-70b` is the whole pitch: $19K of GPU spend carrying a `costCenter` label but no `team`, sitting next to $5K of Claude spend routed through the gateway to Bedrock on an unlabeled legacy key. Two systems, one gap, no existing tool that shows them in the same table.
|
|
52
54
|
|
|
53
55
|
Add a fallback and unalloc says how much of the "allocated" spend only got there through it, so a fallback onto something that is not an owner can't hide inside the headline:
|
|
54
56
|
|
|
@@ -59,11 +61,30 @@ $ unalloc report --fixtures -D team --fallback namespace
|
|
|
59
61
|
│ $23,745.60 attributed only via fallback (namespace) │
|
|
60
62
|
```
|
|
61
63
|
|
|
64
|
+
Then check the ledger against what the providers billed. With `--fixtures`, the bundled sample invoices are used:
|
|
65
|
+
|
|
66
|
+
```
|
|
67
|
+
$ unalloc reconcile --fixtures
|
|
68
|
+
┏━━━━━━━━━━━┳━━━━━━━━━━━━┳━━━━━━━━━━━┳━━━━━━━━━━━━━━━━━━┳━━━━━━━━━━━━┓
|
|
69
|
+
┃ Source ┃ Ledger ┃ Invoice ┃ Delta ┃ Status ┃
|
|
70
|
+
┡━━━━━━━━━━━╇━━━━━━━━━━━━╇━━━━━━━━━━━╇━━━━━━━━━━━━━━━━━━╇━━━━━━━━━━━━┩
|
|
71
|
+
│ anthropic │ $8,737.03 │ $8,779.84 │ $-42.81 (-0.5%) │ matched │
|
|
72
|
+
│ litellm │ $11,444.27 │ — │ — │ no-invoice │
|
|
73
|
+
│ openai │ $4,031.36 │ $4,412.80 │ $-381.44 (-8.6%) │ under │
|
|
74
|
+
│ opencost │ $42,417.72 │ — │ — │ no-invoice │
|
|
75
|
+
└───────────┴────────────┴───────────┴──────────────────┴────────────┘
|
|
76
|
+
The ledger does not tie out. Attribution built on this is not defensible yet.
|
|
77
|
+
```
|
|
78
|
+
|
|
79
|
+
Anthropic agrees within the 1% tolerance. OpenAI billed $381 more than its costs endpoint reported, so any attribution built on that ledger understates OpenAI spend until the gap is explained.
|
|
80
|
+
|
|
62
81
|
## Install
|
|
63
82
|
|
|
64
83
|
```bash
|
|
65
|
-
pip install unalloc
|
|
66
|
-
unalloc report --fixtures
|
|
84
|
+
pip install unalloc # or: uv tool install unalloc
|
|
85
|
+
unalloc report --fixtures --fallback cost_center # works immediately, no infrastructure needed
|
|
86
|
+
unalloc reconcile --fixtures # ledger vs. sample invoices
|
|
87
|
+
unalloc --version
|
|
67
88
|
```
|
|
68
89
|
|
|
69
90
|
From a clone:
|
|
@@ -77,11 +98,11 @@ make check # ruff + pytest, same as CI
|
|
|
77
98
|
Or as a container, which needs no Python on the host:
|
|
78
99
|
|
|
79
100
|
```bash
|
|
80
|
-
docker build -t unalloc
|
|
81
|
-
docker run --rm unalloc
|
|
101
|
+
docker build -t unalloc .
|
|
102
|
+
docker run --rm unalloc report --fixtures -D team
|
|
82
103
|
|
|
83
104
|
# against real systems, reachable from the container
|
|
84
|
-
docker run --rm --network host --env-file .env unalloc
|
|
105
|
+
docker run --rm --network host --env-file .env unalloc report -D team
|
|
85
106
|
```
|
|
86
107
|
|
|
87
108
|
Copy `.env.example` to `.env` for the source configuration. unalloc stores
|
|
@@ -123,7 +144,9 @@ Configure sources by environment variable:
|
|
|
123
144
|
| `ANTHROPIC_ADMIN_KEY` | Anthropic org admin key (cost report endpoint, sent as `x-api-key`) |
|
|
124
145
|
| `UNALLOC_OPENAI_URL`, `UNALLOC_ANTHROPIC_URL` | Optional overrides for the public API roots, e.g. a proxy |
|
|
125
146
|
|
|
126
|
-
Restrict the run with `--source` to avoid double counting: if
|
|
147
|
+
Restrict the run with `--source` to avoid double counting: if LiteLLM covers all provider traffic and OpenCost excludes overlapping provider charges, use `--source opencost --source litellm` and skip the direct provider adapters. Confirm coverage and overlap for your setup. The [end-to-end case study](#case-studies-and-paper) demonstrates the resulting double count with synthetic billing and invoices; it does not measure customer savings.
|
|
148
|
+
|
|
149
|
+
The case-study invoices and monthly cost pools are synthetic or illustrative. The GPU result covers one H100, one model and one two-minute run per load; waiting-request telemetry was sampled about every 0.5 seconds. Allocation-rule disagreement does not establish a uniquely correct charge or production savings.
|
|
127
150
|
|
|
128
151
|
## Reproducing a result
|
|
129
152
|
|
|
@@ -192,11 +215,11 @@ Six studies push real (or realistically simulated) inference workloads through u
|
|
|
192
215
|
| Study | What runs | Headline |
|
|
193
216
|
| --- | --- | --- |
|
|
194
217
|
| [`kv_cache`](https://github.com/timurista/unalloc/blob/main/case_studies/kv_cache) | Discrete-event vLLM-style engine: paged KV blocks, prefix caching, continuous batching, preemption | Step-time metering shows ~0% idle once any request is in flight; KV-memory metering leaves a large share of the bill unowned |
|
|
195
|
-
| [`torch_kv`](https://github.com/timurista/unalloc/blob/main/case_studies/torch_kv) | A from-scratch PyTorch decoder with a real KV cache serving a 4-tenant trace |
|
|
218
|
+
| [`torch_kv`](https://github.com/timurista/unalloc/blob/main/case_studies/torch_kv) | A from-scratch PyTorch decoder with a real KV cache serving a 4-tenant trace | Token and measured-compute allocation rules differ by 33 points for a RAG tenant ($5.7K of an illustrative monthly pool) |
|
|
196
219
|
| [`distributed`](https://github.com/timurista/unalloc/blob/main/case_studies/distributed) | Tensor and pipeline parallel on `torch.distributed` (gloo), verified against a single-process reference | Owner label on LeaderWorkerSet leaders only: 66% unallocated; falling back to `name` "fixes" it by sending 61% to a Helm chart name |
|
|
197
220
|
| [`hybrid_e2e`](https://github.com/timurista/unalloc/blob/main/case_studies/hybrid_e2e) | The real CLI as a subprocess against live mock OpenCost / LiteLLM / OpenAI / Anthropic APIs (Postgres-backed LiteLLM in the dev container) | Enabling every source double counts $11.8K; reading only page one of the billing APIs reports 25% of spend |
|
|
198
221
|
| [`use_cases`](https://github.com/timurista/unalloc/blob/main/case_studies/use_cases) | Labeling Pareto, per-feature unit economics, self-host break-even, CI budget gate | Three label fixes take a 67%-unallocated org under 10% |
|
|
199
|
-
| [`gpu_validation`](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation) | Real vLLM 0.29 serving Qwen2.5-7B on a DigitalOcean H100, driven with the simulator's tenants ([runbook](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation/RUNBOOK.md), [evidence](https://github.com/timurista/unalloc/tree/main/case_studies/results/gpu_validation/evidence)) | Token
|
|
222
|
+
| [`gpu_validation`](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation) | Real vLLM 0.29 serving Qwen2.5-7B on a DigitalOcean H100, driven with the simulator's tenants ([runbook](https://github.com/timurista/unalloc/blob/main/case_studies/gpu_validation/RUNBOOK.md), [evidence](https://github.com/timurista/unalloc/tree/main/case_studies/results/gpu_validation/evidence)) | Token and equal time-share allocation rules differ by 12–14 points for search; GPU "utilization" reads 97% at 2 req/s and 99% at 16 req/s |
|
|
200
223
|
|
|
201
224
|
```bash
|
|
202
225
|
make research # build ./.venv-research: CPU torch, notebook tooling, typst
|
|
@@ -4,7 +4,7 @@ build-backend = "hatchling.build"
|
|
|
4
4
|
|
|
5
5
|
[project]
|
|
6
6
|
name = "unalloc"
|
|
7
|
-
version = "0.3.
|
|
7
|
+
version = "0.3.2"
|
|
8
8
|
description = "Find the AI spend nobody owns. Joins OpenCost allocation data with LLM provider bills and reports what's unattributed."
|
|
9
9
|
readme = "README.md"
|
|
10
10
|
requires-python = ">=3.11"
|
|
@@ -53,6 +53,10 @@ FIXTURE_FILES = {
|
|
|
53
53
|
"anthropic": "anthropic_cost_report.json",
|
|
54
54
|
}
|
|
55
55
|
|
|
56
|
+
# What the providers billed for the fixture month, so `reconcile --fixtures`
|
|
57
|
+
# has something to reconcile against without the reader inventing a number.
|
|
58
|
+
FIXTURE_INVOICES = "invoices.json"
|
|
59
|
+
|
|
56
60
|
ENV_URL = {
|
|
57
61
|
"opencost": "UNALLOC_OPENCOST_URL",
|
|
58
62
|
"litellm": "UNALLOC_LITELLM_URL",
|
|
@@ -67,6 +71,46 @@ ENV_TOKEN = {
|
|
|
67
71
|
}
|
|
68
72
|
|
|
69
73
|
|
|
74
|
+
def _print_version(value: bool) -> None:
|
|
75
|
+
if value:
|
|
76
|
+
typer.echo(__version__)
|
|
77
|
+
raise typer.Exit()
|
|
78
|
+
|
|
79
|
+
|
|
80
|
+
@app.callback()
|
|
81
|
+
def _root(
|
|
82
|
+
version: bool = typer.Option(
|
|
83
|
+
False,
|
|
84
|
+
"--version",
|
|
85
|
+
"-V",
|
|
86
|
+
callback=_print_version,
|
|
87
|
+
is_eager=True,
|
|
88
|
+
help="Print the installed version and exit.",
|
|
89
|
+
),
|
|
90
|
+
) -> None:
|
|
91
|
+
pass
|
|
92
|
+
|
|
93
|
+
|
|
94
|
+
def _fixture_invoices(fixture_dir: Path) -> list[Invoice]:
|
|
95
|
+
path = fixture_dir / FIXTURE_INVOICES
|
|
96
|
+
if not path.exists():
|
|
97
|
+
return []
|
|
98
|
+
data = json.loads(path.read_text())
|
|
99
|
+
period = data.get("period") or {}
|
|
100
|
+
start = datetime.fromisoformat(period["start"])
|
|
101
|
+
end = datetime.fromisoformat(period["end"])
|
|
102
|
+
return [
|
|
103
|
+
Invoice(
|
|
104
|
+
source=item["source"],
|
|
105
|
+
amount_usd=Decimal(str(item["amount_usd"])),
|
|
106
|
+
start=start,
|
|
107
|
+
end=end,
|
|
108
|
+
note=item.get("note", ""),
|
|
109
|
+
)
|
|
110
|
+
for item in data.get("invoices", [])
|
|
111
|
+
]
|
|
112
|
+
|
|
113
|
+
|
|
70
114
|
def _window(days: int) -> tuple[datetime, datetime]:
|
|
71
115
|
end = datetime.now(tz=UTC)
|
|
72
116
|
return end - timedelta(days=days), end
|
|
@@ -281,7 +325,8 @@ def reconcile_cmd(
|
|
|
281
325
|
"--invoice",
|
|
282
326
|
"-i",
|
|
283
327
|
help="Billed total per source as SOURCE=AMOUNT, e.g. openai=4210.55. "
|
|
284
|
-
"Repeat the flag per source."
|
|
328
|
+
"Repeat the flag per source. With --fixtures and no --invoice, the "
|
|
329
|
+
"bundled sample invoices are used.",
|
|
285
330
|
),
|
|
286
331
|
tolerance: float = typer.Option(
|
|
287
332
|
float(DEFAULT_TOLERANCE_PCT),
|
|
@@ -307,6 +352,8 @@ def reconcile_cmd(
|
|
|
307
352
|
end=end,
|
|
308
353
|
)
|
|
309
354
|
)
|
|
355
|
+
if fixtures and not invoice:
|
|
356
|
+
invoices = _fixture_invoices(fixture_dir)
|
|
310
357
|
|
|
311
358
|
render.render_reconciliation(
|
|
312
359
|
reconcile(rows, invoices, tolerance_pct=Decimal(str(tolerance)))
|
|
@@ -0,0 +1,16 @@
|
|
|
1
|
+
# Bundled fixtures
|
|
2
|
+
|
|
3
|
+
Synthetic payloads for `--fixtures`. They describe one month (August 2026) of a
|
|
4
|
+
made-up organization and are built so that **no dollar appears in two sources**:
|
|
5
|
+
|
|
6
|
+
| Source | Covers |
|
|
7
|
+
| --- | --- |
|
|
8
|
+
| `opencost_allocation.json` | Self-hosted GPU and CPU workloads in the cluster (vLLM, llm-d, batch jobs, idle). |
|
|
9
|
+
| `litellm_spend.json` | Gateway traffic routed to Azure OpenAI, Vertex AI and Bedrock. unalloc has no direct adapter for those, so the gateway is their only record. |
|
|
10
|
+
| `openai_costs.json` | Direct OpenAI API usage that does not go through the gateway. |
|
|
11
|
+
| `anthropic_cost_report.json` | Direct Anthropic API usage that does not go through the gateway. |
|
|
12
|
+
| `invoices.json` | What OpenAI and Anthropic billed for the month, used by `reconcile --fixtures`. |
|
|
13
|
+
|
|
14
|
+
Real deployments rarely split this cleanly. If your gateway proxies calls to
|
|
15
|
+
OpenAI or Anthropic, pulling the gateway and the direct provider sources together
|
|
16
|
+
counts that spend twice; choose sources with `--source`.
|
|
@@ -0,0 +1,7 @@
|
|
|
1
|
+
{
|
|
2
|
+
"period": {"start": "2026-08-01T00:00:00Z", "end": "2026-09-01T00:00:00Z"},
|
|
3
|
+
"invoices": [
|
|
4
|
+
{"source": "anthropic", "amount_usd": "8779.84", "note": "August invoice; within rounding of the cost report"},
|
|
5
|
+
{"source": "openai", "amount_usd": "4412.80", "note": "August invoice; $381.44 more than the costs endpoint returned"}
|
|
6
|
+
]
|
|
7
|
+
}
|
|
@@ -1,7 +1,7 @@
|
|
|
1
1
|
{
|
|
2
2
|
"data": [
|
|
3
3
|
{
|
|
4
|
-
"request_id": "req_01", "model": "gpt-4o", "spend": "1840.22",
|
|
4
|
+
"request_id": "req_01", "model": "azure/gpt-4o", "spend": "1840.22",
|
|
5
5
|
"startTime": "2026-08-03T11:02:11Z", "endTime": "2026-08-03T11:02:14Z",
|
|
6
6
|
"prompt_tokens": 812000, "completion_tokens": 240100, "total_tokens": 1052100,
|
|
7
7
|
"api_key_alias": "search-rerank", "team_id": "search",
|
|
@@ -9,7 +9,7 @@
|
|
|
9
9
|
"metadata": {"service": "search-api"}, "cache_hit": false
|
|
10
10
|
},
|
|
11
11
|
{
|
|
12
|
-
"request_id": "req_02", "model": "claude-sonnet-4-6", "spend": "3204.90",
|
|
12
|
+
"request_id": "req_02", "model": "vertex_ai/claude-sonnet-4-6", "spend": "3204.90",
|
|
13
13
|
"startTime": "2026-08-07T09:41:00Z", "endTime": "2026-08-07T09:41:06Z",
|
|
14
14
|
"prompt_tokens": 2140000, "completion_tokens": 512300, "total_tokens": 2652300,
|
|
15
15
|
"api_key_alias": "agent-runtime", "team_id": "platform",
|
|
@@ -17,14 +17,14 @@
|
|
|
17
17
|
"metadata": {"service": "agent-runtime"}, "cache_hit": true
|
|
18
18
|
},
|
|
19
19
|
{
|
|
20
|
-
"request_id": "req_03", "model": "gpt-4o-mini", "spend": "986.40",
|
|
20
|
+
"request_id": "req_03", "model": "azure/gpt-4o-mini", "spend": "986.40",
|
|
21
21
|
"startTime": "2026-08-12T14:20:00Z", "endTime": "2026-08-12T14:20:02Z",
|
|
22
22
|
"prompt_tokens": 9400000, "completion_tokens": 1100000, "total_tokens": 10500000,
|
|
23
23
|
"api_key_alias": "sandbox-shared", "request_tags": [],
|
|
24
24
|
"metadata": {}, "cache_hit": false
|
|
25
25
|
},
|
|
26
26
|
{
|
|
27
|
-
"request_id": "req_04", "model": "claude-opus-5", "spend": "5412.75",
|
|
27
|
+
"request_id": "req_04", "model": "bedrock/claude-opus-5", "spend": "5412.75",
|
|
28
28
|
"startTime": "2026-08-19T17:55:00Z", "endTime": "2026-08-19T17:55:22Z",
|
|
29
29
|
"prompt_tokens": 1802000, "completion_tokens": 940400, "total_tokens": 2742400,
|
|
30
30
|
"api_key_alias": "unknown-legacy-key", "request_tags": ["environment:prod"],
|
|
@@ -0,0 +1,71 @@
|
|
|
1
|
+
from __future__ import annotations
|
|
2
|
+
|
|
3
|
+
import json
|
|
4
|
+
|
|
5
|
+
from typer.testing import CliRunner
|
|
6
|
+
|
|
7
|
+
from unalloc.cli import app
|
|
8
|
+
|
|
9
|
+
runner = CliRunner()
|
|
10
|
+
|
|
11
|
+
|
|
12
|
+
def test_budget_gate_fails_ci_when_unallocated_share_is_too_high():
|
|
13
|
+
# Bundled fixtures are 73.5% unallocated on `team`.
|
|
14
|
+
over = runner.invoke(app, ["report", "--fixtures", "-D", "team", "--json", "--budget", "50"])
|
|
15
|
+
assert over.exit_code == 2
|
|
16
|
+
assert "exceeds budget" in over.stderr
|
|
17
|
+
|
|
18
|
+
under = runner.invoke(app, ["report", "--fixtures", "-D", "team", "--json", "--budget", "80"])
|
|
19
|
+
assert under.exit_code == 0
|
|
20
|
+
assert json.loads(under.stdout)["unallocated_pct"] == "73.5"
|
|
21
|
+
|
|
22
|
+
|
|
23
|
+
def test_labels_backlog_honours_fallbacks():
|
|
24
|
+
plain = runner.invoke(app, ["labels", "--fixtures", "-D", "team", "--limit", "50"])
|
|
25
|
+
rescued = runner.invoke(
|
|
26
|
+
app, ["labels", "--fixtures", "-D", "team", "-f", "namespace", "--limit", "50"]
|
|
27
|
+
)
|
|
28
|
+
assert plain.exit_code == rescued.exit_code == 0
|
|
29
|
+
# Three OpenCost rows lack `team`; two of them (the vLLM pod and the
|
|
30
|
+
# embeddings job) carry a namespace, so only __idle__ stays on the backlog.
|
|
31
|
+
# Count rows rather than match names: rich truncates to terminal width.
|
|
32
|
+
assert plain.stdout.count("│ opencost") == 3
|
|
33
|
+
assert rescued.stdout.count("│ opencost") == 1
|
|
34
|
+
|
|
35
|
+
|
|
36
|
+
def test_version_flag_matches_version_command():
|
|
37
|
+
from unalloc import __version__
|
|
38
|
+
|
|
39
|
+
for args in (["--version"], ["-V"], ["version"]):
|
|
40
|
+
result = runner.invoke(app, args)
|
|
41
|
+
assert result.exit_code == 0
|
|
42
|
+
assert result.stdout.strip() == __version__
|
|
43
|
+
|
|
44
|
+
|
|
45
|
+
def test_reconcile_fixtures_uses_bundled_invoices():
|
|
46
|
+
result = runner.invoke(app, ["reconcile", "--fixtures"])
|
|
47
|
+
assert result.exit_code == 0
|
|
48
|
+
# Anthropic ties out within tolerance; OpenAI billed more than its
|
|
49
|
+
# costs endpoint returned, so the demo shows a real gap.
|
|
50
|
+
assert "matched" in result.stdout
|
|
51
|
+
assert "under" in result.stdout
|
|
52
|
+
assert "does not tie out" in result.stdout
|
|
53
|
+
|
|
54
|
+
|
|
55
|
+
def test_explicit_invoice_replaces_bundled_invoices():
|
|
56
|
+
result = runner.invoke(app, ["reconcile", "--fixtures", "-i", "openai=4031.36"])
|
|
57
|
+
assert result.exit_code == 0
|
|
58
|
+
assert "under" not in result.stdout
|
|
59
|
+
assert "does not tie out" not in result.stdout
|
|
60
|
+
|
|
61
|
+
|
|
62
|
+
def test_fixture_sources_do_not_overlap():
|
|
63
|
+
# The default demo pulls all four sources, so the gateway fixture must not
|
|
64
|
+
# carry traffic that the direct provider fixtures also bill.
|
|
65
|
+
from unalloc.cli import FIXTURE_DIR, FIXTURE_FILES
|
|
66
|
+
|
|
67
|
+
gateway = json.loads((FIXTURE_DIR / FIXTURE_FILES["litellm"]).read_text())["data"]
|
|
68
|
+
direct = ("openai/", "anthropic/")
|
|
69
|
+
for record in gateway:
|
|
70
|
+
model = record["model"]
|
|
71
|
+
assert "/" in model and not model.startswith(direct), model
|
unalloc-0.3.0/tests/test_cli.py
DELETED
|
@@ -1,33 +0,0 @@
|
|
|
1
|
-
from __future__ import annotations
|
|
2
|
-
|
|
3
|
-
import json
|
|
4
|
-
|
|
5
|
-
from typer.testing import CliRunner
|
|
6
|
-
|
|
7
|
-
from unalloc.cli import app
|
|
8
|
-
|
|
9
|
-
runner = CliRunner()
|
|
10
|
-
|
|
11
|
-
|
|
12
|
-
def test_budget_gate_fails_ci_when_unallocated_share_is_too_high():
|
|
13
|
-
# Bundled fixtures are 73.5% unallocated on `team`.
|
|
14
|
-
over = runner.invoke(app, ["report", "--fixtures", "-D", "team", "--json", "--budget", "50"])
|
|
15
|
-
assert over.exit_code == 2
|
|
16
|
-
assert "exceeds budget" in over.stderr
|
|
17
|
-
|
|
18
|
-
under = runner.invoke(app, ["report", "--fixtures", "-D", "team", "--json", "--budget", "80"])
|
|
19
|
-
assert under.exit_code == 0
|
|
20
|
-
assert json.loads(under.stdout)["unallocated_pct"] == "73.5"
|
|
21
|
-
|
|
22
|
-
|
|
23
|
-
def test_labels_backlog_honours_fallbacks():
|
|
24
|
-
plain = runner.invoke(app, ["labels", "--fixtures", "-D", "team", "--limit", "50"])
|
|
25
|
-
rescued = runner.invoke(
|
|
26
|
-
app, ["labels", "--fixtures", "-D", "team", "-f", "namespace", "--limit", "50"]
|
|
27
|
-
)
|
|
28
|
-
assert plain.exit_code == rescued.exit_code == 0
|
|
29
|
-
# Three OpenCost rows lack `team`; two of them (the vLLM pod and the
|
|
30
|
-
# embeddings job) carry a namespace, so only __idle__ stays on the backlog.
|
|
31
|
-
# Count rows rather than match names: rich truncates to terminal width.
|
|
32
|
-
assert plain.stdout.count("│ opencost") == 3
|
|
33
|
-
assert rescued.stdout.count("│ opencost") == 1
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|
|
File without changes
|