barber-llm 0.1.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- barber_llm-0.1.0/.gitignore +10 -0
- barber_llm-0.1.0/LICENSE +21 -0
- barber_llm-0.1.0/PKG-INFO +247 -0
- barber_llm-0.1.0/README.md +217 -0
- barber_llm-0.1.0/barber/__init__.py +96 -0
- barber_llm-0.1.0/barber/core.py +303 -0
- barber_llm-0.1.0/barber/embedders.py +79 -0
- barber_llm-0.1.0/barber/eval/JUDGE_PROMPT.md +101 -0
- barber_llm-0.1.0/barber/eval/__init__.py +9 -0
- barber_llm-0.1.0/barber/eval/harness.py +324 -0
- barber_llm-0.1.0/docs/methodology.md +126 -0
- barber_llm-0.1.0/pyproject.toml +44 -0
- barber_llm-0.1.0/tests/test_determinism.py +70 -0
- barber_llm-0.1.0/tests/test_pinning.py +58 -0
- barber_llm-0.1.0/tests/test_trim.py +129 -0
barber_llm-0.1.0/LICENSE
ADDED
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Nadir (getnadir.com)
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,247 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: barber-llm
|
|
3
|
+
Version: 0.1.0
|
|
4
|
+
Summary: Query-aware context trimming for LLM requests. Your context could use a trim.
|
|
5
|
+
Project-URL: Homepage, https://getnadir.com
|
|
6
|
+
Project-URL: Source, https://github.com/NadirRouter/barber
|
|
7
|
+
Author: Nadir
|
|
8
|
+
License-Expression: MIT
|
|
9
|
+
License-File: LICENSE
|
|
10
|
+
Keywords: context,cost,llm,prompt-compression,rag,tokens
|
|
11
|
+
Classifier: Development Status :: 4 - Beta
|
|
12
|
+
Classifier: Intended Audience :: Developers
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
18
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
19
|
+
Classifier: Topic :: Software Development :: Libraries :: Python Modules
|
|
20
|
+
Requires-Python: >=3.10
|
|
21
|
+
Provides-Extra: eval
|
|
22
|
+
Requires-Dist: datasets; extra == 'eval'
|
|
23
|
+
Requires-Dist: openai; extra == 'eval'
|
|
24
|
+
Requires-Dist: tiktoken; extra == 'eval'
|
|
25
|
+
Provides-Extra: semantic
|
|
26
|
+
Requires-Dist: sentence-transformers; extra == 'semantic'
|
|
27
|
+
Provides-Extra: tokens
|
|
28
|
+
Requires-Dist: tiktoken; extra == 'tokens'
|
|
29
|
+
Description-Content-Type: text/markdown
|
|
30
|
+
|
|
31
|
+
<p align="center">
|
|
32
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-readme-hero.png" alt="barber: your context could use a trim" width="100%">
|
|
33
|
+
</p>
|
|
34
|
+
|
|
35
|
+
# barber
|
|
36
|
+
|
|
37
|
+
Query-aware context trimming for LLM requests.
|
|
38
|
+
|
|
39
|
+
**Your context could use a trim.** *33% off the top.*
|
|
40
|
+
|
|
41
|
+
[](https://pypi.org/project/barber-llm/)
|
|
42
|
+
[](https://pypi.org/project/barber-llm/)
|
|
43
|
+
[](https://github.com/NadirRouter/barber/blob/main/LICENSE)
|
|
44
|
+
[](https://github.com/NadirRouter/barber/blob/main/pyproject.toml)
|
|
45
|
+
|
|
46
|
+
## The 30-second pitch
|
|
47
|
+
|
|
48
|
+
Retrieved context is mostly irrelevant to any single question. Your RAG stack
|
|
49
|
+
fetches ten paragraphs, the question needs two, and you pay for all ten on
|
|
50
|
+
every request.
|
|
51
|
+
|
|
52
|
+
barber embeds the chunks and the question, keeps what matters, and drops the
|
|
53
|
+
rest. Nothing is rewritten or summarized: chunks survive verbatim or vanish.
|
|
54
|
+
No model calls at trim time, no required dependencies, deterministic output.
|
|
55
|
+
|
|
56
|
+
In the published benchmark that locked barber's defaults, that meant 31.8 to
|
|
57
|
+
34.1% of context tokens gone with answer quality within noise of full context
|
|
58
|
+
([numbers below](#the-benchmark)).
|
|
59
|
+
|
|
60
|
+
## Install
|
|
61
|
+
|
|
62
|
+
```bash
|
|
63
|
+
pip install barber-llm
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The PyPI name is `barber-llm`; the import name is `barber`. Zero required
|
|
67
|
+
dependencies. Extras when you want them:
|
|
68
|
+
|
|
69
|
+
| Extra | Pulls in | Gives you |
|
|
70
|
+
|---|---|---|
|
|
71
|
+
| `barber-llm[semantic]` | `sentence-transformers` | semantic scoring, paraphrase-safe |
|
|
72
|
+
| `barber-llm[tokens]` | `tiktoken` | exact token counts in `TrimResult` |
|
|
73
|
+
| `barber-llm[eval]` | `datasets`, `openai`, `tiktoken` | the `barber-eval` benchmark harness |
|
|
74
|
+
|
|
75
|
+
## Quickstart
|
|
76
|
+
|
|
77
|
+
Zero-dependency lexical mode:
|
|
78
|
+
|
|
79
|
+
```python
|
|
80
|
+
from barber import trim
|
|
81
|
+
|
|
82
|
+
result = trim(messages) # OpenAI-style list[dict], keep=0.6
|
|
83
|
+
send_to_your_llm(result.messages) # same conversation, fewer tokens
|
|
84
|
+
print(result.tokens_saved, result.chunks_dropped, result.changed)
|
|
85
|
+
```
|
|
86
|
+
|
|
87
|
+
Semantic mode, the configuration the benchmark shipped with:
|
|
88
|
+
|
|
89
|
+
```python
|
|
90
|
+
from barber import trim, embedders
|
|
91
|
+
|
|
92
|
+
embed = embedders.sentence_transformers("llm-semantic-router/mmbert-embed-32k-2d-matryoshka")
|
|
93
|
+
result = trim(messages, keep=0.6, embedder=embed)
|
|
94
|
+
```
|
|
95
|
+
|
|
96
|
+
The lexical fallback is deterministic and dependency-free, but it matches
|
|
97
|
+
words, not meaning. For production traffic use the semantic embedder above
|
|
98
|
+
(32K context window) or `BAAI/bge-small-en-v1.5` as the lightweight
|
|
99
|
+
alternative; the two tied on quality in the published runs. There is also
|
|
100
|
+
`embedders.endpoint(base_url, model, api_key)` for any OpenAI compatible
|
|
101
|
+
`/v1/embeddings` server (vLLM, TEI), which needs the `openai` package.
|
|
102
|
+
|
|
103
|
+
Multi-turn pipelines use the transform form. Pass a shared cache and a block is
|
|
104
|
+
decided once, then replayed byte-identically on every later turn, so your
|
|
105
|
+
provider prompt cache stays warm:
|
|
106
|
+
|
|
107
|
+
```python
|
|
108
|
+
from barber import make_transform
|
|
109
|
+
|
|
110
|
+
name, fn = make_transform(embedder=embed, keep=0.6, cache=shared_dict)
|
|
111
|
+
messages, changed = fn(messages) # call this every turn
|
|
112
|
+
```
|
|
113
|
+
|
|
114
|
+
## The benchmark
|
|
115
|
+
|
|
116
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-benchmark-results.png" alt="benchmark stat card" align="right" width="300">
|
|
117
|
+
|
|
118
|
+
barber's defaults were locked by a paired A/B on HotpotQA (distractor config):
|
|
119
|
+
answer each question with full context and with trimmed context, then have a
|
|
120
|
+
blind judge grade both answers against the gold reference.
|
|
121
|
+
|
|
122
|
+
| | Medium (~6K tok) | Large (~14K tok) |
|
|
123
|
+
|---|---|---|
|
|
124
|
+
| Tokens saved | 31.8% | 34.1% |
|
|
125
|
+
| Answer-paragraph retention | 100% | 100% |
|
|
126
|
+
| Full-context accuracy | 97.2% | 94.8% |
|
|
127
|
+
| Trimmed-context accuracy | 96.0% | 95.9% |
|
|
128
|
+
|
|
129
|
+
About 350 judged pairs, MiniMax M3 as the blind judge, answers graded against
|
|
130
|
+
gold references, `keep=0.6`. On large contexts trimmed beat full: selection
|
|
131
|
+
removes the distractors models trip over.
|
|
132
|
+
|
|
133
|
+
Full write-up: [the benchmark post](https://getnadir.com/blog/context-selection-benchmark-minimax.html).
|
|
134
|
+
Full protocol: [docs/methodology.md](https://github.com/NadirRouter/barber/blob/main/docs/methodology.md). Reproduce it:
|
|
135
|
+
|
|
136
|
+
```bash
|
|
137
|
+
pip install "barber-llm[eval]" && barber-eval --n 200 --keep 0.6 --size large
|
|
138
|
+
```
|
|
139
|
+
|
|
140
|
+
<br clear="right">
|
|
141
|
+
|
|
142
|
+
## How it works
|
|
143
|
+
|
|
144
|
+
<p align="center">
|
|
145
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-how-it-works.png" alt="how barber works" width="100%">
|
|
146
|
+
</p>
|
|
147
|
+
|
|
148
|
+
1. **Query.** The latest user message is the question. It is never trimmed.
|
|
149
|
+
2. **Candidate blocks.** Large user, tool, and function messages (800+ chars,
|
|
150
|
+
4+ chunks). System and assistant messages are never candidates.
|
|
151
|
+
3. **Chunk.** Split each block on blank lines, `---` rules, headings, and
|
|
152
|
+
`[n]` citation markers, the boundaries RAG concatenations already have.
|
|
153
|
+
Falls back to sentences for a single wall of prose.
|
|
154
|
+
4. **Score.** Embed every chunk plus the question, cosine each chunk against
|
|
155
|
+
the question, keep the top `keep` fraction.
|
|
156
|
+
5. **Guards.** Pinned chunks bypass the budget entirely, the first and last
|
|
157
|
+
chunk always survive, and a relevance floor drops pure noise even when the
|
|
158
|
+
budget would keep it. Details below.
|
|
159
|
+
6. **Marker.** Each dropped run collapses into one line, so the model knows
|
|
160
|
+
the cut happened and doesn't go looking for missing text:
|
|
161
|
+
|
|
162
|
+
```
|
|
163
|
+
[… 4 passage(s) omitted as not relevant to this question — the remaining context is sufficient …]
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
The assertive wording is deliberate and benchmark-locked. A neutral
|
|
167
|
+
"lower-relevance passages omitted" primes refusals; this one tells the
|
|
168
|
+
model to proceed. Re-run that ablation yourself with
|
|
169
|
+
`barber-eval --marker neutral`.
|
|
170
|
+
|
|
171
|
+
Decisions are memoized on the block hash alone, never the query. The first
|
|
172
|
+
turn to see a block decides it; every later turn replays the decision
|
|
173
|
+
byte-identically. Your history prefix stays stable and your provider prompt
|
|
174
|
+
cache keeps hitting.
|
|
175
|
+
|
|
176
|
+
## What barber does not do
|
|
177
|
+
|
|
178
|
+
- **No summarization.** Chunks survive verbatim or vanish. A summary is a
|
|
179
|
+
rewrite, and rewrites can silently invent or lose facts.
|
|
180
|
+
- **No letter tricks.** Dropping vowels, truncating words, gzip-then-base64:
|
|
181
|
+
in the same benchmark, every character-level scheme cost MORE tokens, not
|
|
182
|
+
fewer. Letter removal measured 1.34x to 1.69x the tokens; classic
|
|
183
|
+
compression 3.2x to 3.7x. Tokenizers already are compressors; fighting them
|
|
184
|
+
backfires. [The numbers](https://getnadir.com/blog/context-selection-benchmark-minimax.html).
|
|
185
|
+
- **No history compaction.** Providers do that natively now, and re-writing
|
|
186
|
+
old turns busts their prompt caches. barber targets fresh retrieved context
|
|
187
|
+
only.
|
|
188
|
+
- **No model calls at trim time.** Scoring is an embedding pass (or pure
|
|
189
|
+
lexical math). Nothing is sent anywhere unless you opt into the
|
|
190
|
+
`endpoint()` embedder.
|
|
191
|
+
|
|
192
|
+
## Guards, or why your answers survive
|
|
193
|
+
|
|
194
|
+
The failure mode of context pruning is silent: drop the one chunk that held
|
|
195
|
+
the answer and nothing errors, the model just answers worse. barber ships with
|
|
196
|
+
every guard on:
|
|
197
|
+
|
|
198
|
+
- **Deontic and PII pinning.** Chunks with constraint language ("must",
|
|
199
|
+
"never", "do not", "prohibited", "required", "only if") or sensitive-data
|
|
200
|
+
markers (PII, HIPAA, PCI, SSN, password, secret, API key) are never dropped.
|
|
201
|
+
- **Rare-query-entity pinning.** A query term that appears in only one or two
|
|
202
|
+
chunks of a block is a strong "this chunk answers the question" signal.
|
|
203
|
+
Those chunks are never dropped, which is what protects multi-hop questions.
|
|
204
|
+
- **Lead and tail keep.** The first and last chunk of every block survive:
|
|
205
|
+
headers, conclusions, and the lost-in-the-middle mitigation.
|
|
206
|
+
- **Relevance floor (0.35).** A chunk scoring far below the block's top chunk
|
|
207
|
+
is noise even if the budget has room for it.
|
|
208
|
+
- **Never touched at all:** the latest user message, system messages,
|
|
209
|
+
assistant messages, any message under 800 chars, any block under 4 chunks.
|
|
210
|
+
|
|
211
|
+
In the benchmark these guards held answer-paragraph retention at 100% in both
|
|
212
|
+
published sizes ([table above](#the-benchmark)).
|
|
213
|
+
|
|
214
|
+
## Evaluate it on your data
|
|
215
|
+
|
|
216
|
+
Published numbers are an existence proof, not a guarantee for your traffic.
|
|
217
|
+
The harness that produced them ships in the package:
|
|
218
|
+
|
|
219
|
+
```bash
|
|
220
|
+
pip install "barber-llm[eval]"
|
|
221
|
+
export JUDGE_API_KEY=sk-... # any OpenAI compatible judge; MiniMax by default
|
|
222
|
+
|
|
223
|
+
barber-eval --baseline --n 25 # sanity-check generator + judge first
|
|
224
|
+
barber-eval --n 200 --keep 0.6 --size large
|
|
225
|
+
```
|
|
226
|
+
|
|
227
|
+
The number to gate on is the **regression rate**: how often trimming turned a
|
|
228
|
+
right answer wrong. Hold it under 1 to 2%, then take the most aggressive
|
|
229
|
+
`keep` that stays under your bar. Tokens saved is the payoff, not the gate.
|
|
230
|
+
The exact judge prompt ships in
|
|
231
|
+
[barber/eval/JUDGE_PROMPT.md](https://github.com/NadirRouter/barber/blob/main/barber/eval/JUDGE_PROMPT.md), including a
|
|
232
|
+
reference-free variant for chat data without gold answers. Judge and generator
|
|
233
|
+
are swappable via `JUDGE_*` and `GEN_*` env vars
|
|
234
|
+
([details](https://github.com/NadirRouter/barber/blob/main/docs/methodology.md)).
|
|
235
|
+
|
|
236
|
+
## Credits
|
|
237
|
+
|
|
238
|
+
- The **MiniMax team**: MiniMax M3 was the generator and the blind judge for
|
|
239
|
+
the entire eval, about 2,000 model calls on well under $20 of credit.
|
|
240
|
+
- The **vLLM Semantic Router team**: their
|
|
241
|
+
[mmbert-embed-32k-2d-matryoshka](https://huggingface.co/llm-semantic-router/mmbert-embed-32k-2d-matryoshka)
|
|
242
|
+
is the recommended embedder, picked for its 32K window.
|
|
243
|
+
|
|
244
|
+
---
|
|
245
|
+
|
|
246
|
+
Want this tier-aware, quality-monitored in production, and billed only on
|
|
247
|
+
verified savings? That's [Nadir](https://getnadir.com).
|
|
@@ -0,0 +1,217 @@
|
|
|
1
|
+
<p align="center">
|
|
2
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-readme-hero.png" alt="barber: your context could use a trim" width="100%">
|
|
3
|
+
</p>
|
|
4
|
+
|
|
5
|
+
# barber
|
|
6
|
+
|
|
7
|
+
Query-aware context trimming for LLM requests.
|
|
8
|
+
|
|
9
|
+
**Your context could use a trim.** *33% off the top.*
|
|
10
|
+
|
|
11
|
+
[](https://pypi.org/project/barber-llm/)
|
|
12
|
+
[](https://pypi.org/project/barber-llm/)
|
|
13
|
+
[](https://github.com/NadirRouter/barber/blob/main/LICENSE)
|
|
14
|
+
[](https://github.com/NadirRouter/barber/blob/main/pyproject.toml)
|
|
15
|
+
|
|
16
|
+
## The 30-second pitch
|
|
17
|
+
|
|
18
|
+
Retrieved context is mostly irrelevant to any single question. Your RAG stack
|
|
19
|
+
fetches ten paragraphs, the question needs two, and you pay for all ten on
|
|
20
|
+
every request.
|
|
21
|
+
|
|
22
|
+
barber embeds the chunks and the question, keeps what matters, and drops the
|
|
23
|
+
rest. Nothing is rewritten or summarized: chunks survive verbatim or vanish.
|
|
24
|
+
No model calls at trim time, no required dependencies, deterministic output.
|
|
25
|
+
|
|
26
|
+
In the published benchmark that locked barber's defaults, that meant 31.8 to
|
|
27
|
+
34.1% of context tokens gone with answer quality within noise of full context
|
|
28
|
+
([numbers below](#the-benchmark)).
|
|
29
|
+
|
|
30
|
+
## Install
|
|
31
|
+
|
|
32
|
+
```bash
|
|
33
|
+
pip install barber-llm
|
|
34
|
+
```
|
|
35
|
+
|
|
36
|
+
The PyPI name is `barber-llm`; the import name is `barber`. Zero required
|
|
37
|
+
dependencies. Extras when you want them:
|
|
38
|
+
|
|
39
|
+
| Extra | Pulls in | Gives you |
|
|
40
|
+
|---|---|---|
|
|
41
|
+
| `barber-llm[semantic]` | `sentence-transformers` | semantic scoring, paraphrase-safe |
|
|
42
|
+
| `barber-llm[tokens]` | `tiktoken` | exact token counts in `TrimResult` |
|
|
43
|
+
| `barber-llm[eval]` | `datasets`, `openai`, `tiktoken` | the `barber-eval` benchmark harness |
|
|
44
|
+
|
|
45
|
+
## Quickstart
|
|
46
|
+
|
|
47
|
+
Zero-dependency lexical mode:
|
|
48
|
+
|
|
49
|
+
```python
|
|
50
|
+
from barber import trim
|
|
51
|
+
|
|
52
|
+
result = trim(messages) # OpenAI-style list[dict], keep=0.6
|
|
53
|
+
send_to_your_llm(result.messages) # same conversation, fewer tokens
|
|
54
|
+
print(result.tokens_saved, result.chunks_dropped, result.changed)
|
|
55
|
+
```
|
|
56
|
+
|
|
57
|
+
Semantic mode, the configuration the benchmark shipped with:
|
|
58
|
+
|
|
59
|
+
```python
|
|
60
|
+
from barber import trim, embedders
|
|
61
|
+
|
|
62
|
+
embed = embedders.sentence_transformers("llm-semantic-router/mmbert-embed-32k-2d-matryoshka")
|
|
63
|
+
result = trim(messages, keep=0.6, embedder=embed)
|
|
64
|
+
```
|
|
65
|
+
|
|
66
|
+
The lexical fallback is deterministic and dependency-free, but it matches
|
|
67
|
+
words, not meaning. For production traffic use the semantic embedder above
|
|
68
|
+
(32K context window) or `BAAI/bge-small-en-v1.5` as the lightweight
|
|
69
|
+
alternative; the two tied on quality in the published runs. There is also
|
|
70
|
+
`embedders.endpoint(base_url, model, api_key)` for any OpenAI compatible
|
|
71
|
+
`/v1/embeddings` server (vLLM, TEI), which needs the `openai` package.
|
|
72
|
+
|
|
73
|
+
Multi-turn pipelines use the transform form. Pass a shared cache and a block is
|
|
74
|
+
decided once, then replayed byte-identically on every later turn, so your
|
|
75
|
+
provider prompt cache stays warm:
|
|
76
|
+
|
|
77
|
+
```python
|
|
78
|
+
from barber import make_transform
|
|
79
|
+
|
|
80
|
+
name, fn = make_transform(embedder=embed, keep=0.6, cache=shared_dict)
|
|
81
|
+
messages, changed = fn(messages) # call this every turn
|
|
82
|
+
```
|
|
83
|
+
|
|
84
|
+
## The benchmark
|
|
85
|
+
|
|
86
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-benchmark-results.png" alt="benchmark stat card" align="right" width="300">
|
|
87
|
+
|
|
88
|
+
barber's defaults were locked by a paired A/B on HotpotQA (distractor config):
|
|
89
|
+
answer each question with full context and with trimmed context, then have a
|
|
90
|
+
blind judge grade both answers against the gold reference.
|
|
91
|
+
|
|
92
|
+
| | Medium (~6K tok) | Large (~14K tok) |
|
|
93
|
+
|---|---|---|
|
|
94
|
+
| Tokens saved | 31.8% | 34.1% |
|
|
95
|
+
| Answer-paragraph retention | 100% | 100% |
|
|
96
|
+
| Full-context accuracy | 97.2% | 94.8% |
|
|
97
|
+
| Trimmed-context accuracy | 96.0% | 95.9% |
|
|
98
|
+
|
|
99
|
+
About 350 judged pairs, MiniMax M3 as the blind judge, answers graded against
|
|
100
|
+
gold references, `keep=0.6`. On large contexts trimmed beat full: selection
|
|
101
|
+
removes the distractors models trip over.
|
|
102
|
+
|
|
103
|
+
Full write-up: [the benchmark post](https://getnadir.com/blog/context-selection-benchmark-minimax.html).
|
|
104
|
+
Full protocol: [docs/methodology.md](https://github.com/NadirRouter/barber/blob/main/docs/methodology.md). Reproduce it:
|
|
105
|
+
|
|
106
|
+
```bash
|
|
107
|
+
pip install "barber-llm[eval]" && barber-eval --n 200 --keep 0.6 --size large
|
|
108
|
+
```
|
|
109
|
+
|
|
110
|
+
<br clear="right">
|
|
111
|
+
|
|
112
|
+
## How it works
|
|
113
|
+
|
|
114
|
+
<p align="center">
|
|
115
|
+
<img src="https://raw.githubusercontent.com/NadirRouter/barber/main/assets/barber-how-it-works.png" alt="how barber works" width="100%">
|
|
116
|
+
</p>
|
|
117
|
+
|
|
118
|
+
1. **Query.** The latest user message is the question. It is never trimmed.
|
|
119
|
+
2. **Candidate blocks.** Large user, tool, and function messages (800+ chars,
|
|
120
|
+
4+ chunks). System and assistant messages are never candidates.
|
|
121
|
+
3. **Chunk.** Split each block on blank lines, `---` rules, headings, and
|
|
122
|
+
`[n]` citation markers, the boundaries RAG concatenations already have.
|
|
123
|
+
Falls back to sentences for a single wall of prose.
|
|
124
|
+
4. **Score.** Embed every chunk plus the question, cosine each chunk against
|
|
125
|
+
the question, keep the top `keep` fraction.
|
|
126
|
+
5. **Guards.** Pinned chunks bypass the budget entirely, the first and last
|
|
127
|
+
chunk always survive, and a relevance floor drops pure noise even when the
|
|
128
|
+
budget would keep it. Details below.
|
|
129
|
+
6. **Marker.** Each dropped run collapses into one line, so the model knows
|
|
130
|
+
the cut happened and doesn't go looking for missing text:
|
|
131
|
+
|
|
132
|
+
```
|
|
133
|
+
[… 4 passage(s) omitted as not relevant to this question — the remaining context is sufficient …]
|
|
134
|
+
```
|
|
135
|
+
|
|
136
|
+
The assertive wording is deliberate and benchmark-locked. A neutral
|
|
137
|
+
"lower-relevance passages omitted" primes refusals; this one tells the
|
|
138
|
+
model to proceed. Re-run that ablation yourself with
|
|
139
|
+
`barber-eval --marker neutral`.
|
|
140
|
+
|
|
141
|
+
Decisions are memoized on the block hash alone, never the query. The first
|
|
142
|
+
turn to see a block decides it; every later turn replays the decision
|
|
143
|
+
byte-identically. Your history prefix stays stable and your provider prompt
|
|
144
|
+
cache keeps hitting.
|
|
145
|
+
|
|
146
|
+
## What barber does not do
|
|
147
|
+
|
|
148
|
+
- **No summarization.** Chunks survive verbatim or vanish. A summary is a
|
|
149
|
+
rewrite, and rewrites can silently invent or lose facts.
|
|
150
|
+
- **No letter tricks.** Dropping vowels, truncating words, gzip-then-base64:
|
|
151
|
+
in the same benchmark, every character-level scheme cost MORE tokens, not
|
|
152
|
+
fewer. Letter removal measured 1.34x to 1.69x the tokens; classic
|
|
153
|
+
compression 3.2x to 3.7x. Tokenizers already are compressors; fighting them
|
|
154
|
+
backfires. [The numbers](https://getnadir.com/blog/context-selection-benchmark-minimax.html).
|
|
155
|
+
- **No history compaction.** Providers do that natively now, and re-writing
|
|
156
|
+
old turns busts their prompt caches. barber targets fresh retrieved context
|
|
157
|
+
only.
|
|
158
|
+
- **No model calls at trim time.** Scoring is an embedding pass (or pure
|
|
159
|
+
lexical math). Nothing is sent anywhere unless you opt into the
|
|
160
|
+
`endpoint()` embedder.
|
|
161
|
+
|
|
162
|
+
## Guards, or why your answers survive
|
|
163
|
+
|
|
164
|
+
The failure mode of context pruning is silent: drop the one chunk that held
|
|
165
|
+
the answer and nothing errors, the model just answers worse. barber ships with
|
|
166
|
+
every guard on:
|
|
167
|
+
|
|
168
|
+
- **Deontic and PII pinning.** Chunks with constraint language ("must",
|
|
169
|
+
"never", "do not", "prohibited", "required", "only if") or sensitive-data
|
|
170
|
+
markers (PII, HIPAA, PCI, SSN, password, secret, API key) are never dropped.
|
|
171
|
+
- **Rare-query-entity pinning.** A query term that appears in only one or two
|
|
172
|
+
chunks of a block is a strong "this chunk answers the question" signal.
|
|
173
|
+
Those chunks are never dropped, which is what protects multi-hop questions.
|
|
174
|
+
- **Lead and tail keep.** The first and last chunk of every block survive:
|
|
175
|
+
headers, conclusions, and the lost-in-the-middle mitigation.
|
|
176
|
+
- **Relevance floor (0.35).** A chunk scoring far below the block's top chunk
|
|
177
|
+
is noise even if the budget has room for it.
|
|
178
|
+
- **Never touched at all:** the latest user message, system messages,
|
|
179
|
+
assistant messages, any message under 800 chars, any block under 4 chunks.
|
|
180
|
+
|
|
181
|
+
In the benchmark these guards held answer-paragraph retention at 100% in both
|
|
182
|
+
published sizes ([table above](#the-benchmark)).
|
|
183
|
+
|
|
184
|
+
## Evaluate it on your data
|
|
185
|
+
|
|
186
|
+
Published numbers are an existence proof, not a guarantee for your traffic.
|
|
187
|
+
The harness that produced them ships in the package:
|
|
188
|
+
|
|
189
|
+
```bash
|
|
190
|
+
pip install "barber-llm[eval]"
|
|
191
|
+
export JUDGE_API_KEY=sk-... # any OpenAI compatible judge; MiniMax by default
|
|
192
|
+
|
|
193
|
+
barber-eval --baseline --n 25 # sanity-check generator + judge first
|
|
194
|
+
barber-eval --n 200 --keep 0.6 --size large
|
|
195
|
+
```
|
|
196
|
+
|
|
197
|
+
The number to gate on is the **regression rate**: how often trimming turned a
|
|
198
|
+
right answer wrong. Hold it under 1 to 2%, then take the most aggressive
|
|
199
|
+
`keep` that stays under your bar. Tokens saved is the payoff, not the gate.
|
|
200
|
+
The exact judge prompt ships in
|
|
201
|
+
[barber/eval/JUDGE_PROMPT.md](https://github.com/NadirRouter/barber/blob/main/barber/eval/JUDGE_PROMPT.md), including a
|
|
202
|
+
reference-free variant for chat data without gold answers. Judge and generator
|
|
203
|
+
are swappable via `JUDGE_*` and `GEN_*` env vars
|
|
204
|
+
([details](https://github.com/NadirRouter/barber/blob/main/docs/methodology.md)).
|
|
205
|
+
|
|
206
|
+
## Credits
|
|
207
|
+
|
|
208
|
+
- The **MiniMax team**: MiniMax M3 was the generator and the blind judge for
|
|
209
|
+
the entire eval, about 2,000 model calls on well under $20 of credit.
|
|
210
|
+
- The **vLLM Semantic Router team**: their
|
|
211
|
+
[mmbert-embed-32k-2d-matryoshka](https://huggingface.co/llm-semantic-router/mmbert-embed-32k-2d-matryoshka)
|
|
212
|
+
is the recommended embedder, picked for its 32K window.
|
|
213
|
+
|
|
214
|
+
---
|
|
215
|
+
|
|
216
|
+
Want this tier-aware, quality-monitored in production, and billed only on
|
|
217
|
+
verified savings? That's [Nadir](https://getnadir.com).
|
|
@@ -0,0 +1,96 @@
|
|
|
1
|
+
"""barber — query-aware context trimming for LLM requests.
|
|
2
|
+
|
|
3
|
+
Your context could use a trim.
|
|
4
|
+
|
|
5
|
+
from barber import trim
|
|
6
|
+
|
|
7
|
+
result = trim(messages, keep=0.6)
|
|
8
|
+
send(result.messages) # same conversation, fewer tokens
|
|
9
|
+
|
|
10
|
+
Chunks survive verbatim or vanish. Nothing is rewritten, nothing is summarized,
|
|
11
|
+
and no model is called at trim time. The selection algorithm is the validated,
|
|
12
|
+
benchmarked one from Nadir (see README for the numbers).
|
|
13
|
+
"""
|
|
14
|
+
from __future__ import annotations
|
|
15
|
+
from dataclasses import dataclass, replace
|
|
16
|
+
from typing import Callable, Optional
|
|
17
|
+
|
|
18
|
+
from .core import SelectionConfig, SelectionStats, make_selection_transform
|
|
19
|
+
from .core import _text_of
|
|
20
|
+
|
|
21
|
+
__version__ = "0.1.0"
|
|
22
|
+
|
|
23
|
+
__all__ = ["trim", "TrimResult", "make_transform", "SelectionConfig", "__version__"]
|
|
24
|
+
|
|
25
|
+
|
|
26
|
+
@dataclass
|
|
27
|
+
class TrimResult:
|
|
28
|
+
messages: list # trimmed messages (originals are not mutated)
|
|
29
|
+
tokens_saved: int # tiktoken o200k count if installed, len//4 fallback
|
|
30
|
+
chunks_dropped: int
|
|
31
|
+
changed: bool
|
|
32
|
+
|
|
33
|
+
|
|
34
|
+
def _token_counter() -> Callable[[str], int]:
|
|
35
|
+
try:
|
|
36
|
+
import tiktoken
|
|
37
|
+
enc = tiktoken.get_encoding("o200k_base")
|
|
38
|
+
return lambda s: len(enc.encode(s))
|
|
39
|
+
except Exception:
|
|
40
|
+
return lambda s: len(s) // 4
|
|
41
|
+
|
|
42
|
+
|
|
43
|
+
def _count_tokens(messages: list, ntok: Callable[[str], int]) -> int:
|
|
44
|
+
return sum(ntok(_text_of(m.get("content"))) for m in messages)
|
|
45
|
+
|
|
46
|
+
|
|
47
|
+
def make_transform(
|
|
48
|
+
embedder: Optional[Callable] = None,
|
|
49
|
+
keep: float = 0.6,
|
|
50
|
+
*,
|
|
51
|
+
cfg: Optional[SelectionConfig] = None,
|
|
52
|
+
cache: Optional[dict] = None,
|
|
53
|
+
):
|
|
54
|
+
"""Return the ("barber", fn) pair for pipeline integration, where
|
|
55
|
+
fn(messages) -> (messages, changed).
|
|
56
|
+
|
|
57
|
+
Pass a shared `cache` dict to get freeze-on-first-sight memoization across
|
|
58
|
+
turns: the first turn to see a block decides it, every later turn replays
|
|
59
|
+
that decision byte-identically, keeping the provider prompt cache warm.
|
|
60
|
+
`keep` pins the fraction of chunks retained (0.6 is the benchmark default).
|
|
61
|
+
"""
|
|
62
|
+
cfg = replace(cfg or SelectionConfig(), min_keep_ratio=keep, max_keep_ratio=keep)
|
|
63
|
+
_, fn = make_selection_transform(embed_fn=embedder, cfg=cfg, decision_cache=cache)
|
|
64
|
+
return ("barber", fn)
|
|
65
|
+
|
|
66
|
+
|
|
67
|
+
def trim(
|
|
68
|
+
messages: list,
|
|
69
|
+
keep: float = 0.6,
|
|
70
|
+
embedder: Optional[Callable] = None,
|
|
71
|
+
*,
|
|
72
|
+
cfg: Optional[SelectionConfig] = None,
|
|
73
|
+
cache: Optional[dict] = None,
|
|
74
|
+
) -> TrimResult:
|
|
75
|
+
"""Trim query-irrelevant chunks out of an OpenAI-style message list.
|
|
76
|
+
|
|
77
|
+
messages: list[dict] with "role" and "content" keys.
|
|
78
|
+
keep: fraction of chunks retained per block (benchmark default 0.6).
|
|
79
|
+
embedder: None -> deterministic lexical fallback (zero deps), or one of
|
|
80
|
+
barber.embedders.lexical / sentence_transformers / endpoint.
|
|
81
|
+
|
|
82
|
+
The latest user message and system messages are never trimmed. Guards
|
|
83
|
+
(lead/tail keep, deontic/PII pinning, rare-query-entity pinning, relevance
|
|
84
|
+
floor) are always on. Deterministic: same input, same output.
|
|
85
|
+
"""
|
|
86
|
+
_, fn = make_transform(embedder, keep, cfg=cfg, cache=cache)
|
|
87
|
+
out, changed = fn(list(messages))
|
|
88
|
+
stats: SelectionStats = fn.last_stats
|
|
89
|
+
ntok = _token_counter()
|
|
90
|
+
saved = _count_tokens(messages, ntok) - _count_tokens(out, ntok)
|
|
91
|
+
return TrimResult(
|
|
92
|
+
messages=out,
|
|
93
|
+
tokens_saved=max(0, saved),
|
|
94
|
+
chunks_dropped=stats.chunks_in - stats.chunks_kept,
|
|
95
|
+
changed=changed,
|
|
96
|
+
)
|