llm-cheap-filter 0.2.0__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- llm_cheap_filter-0.2.0/LICENSE +21 -0
- llm_cheap_filter-0.2.0/PKG-INFO +249 -0
- llm_cheap_filter-0.2.0/README.md +229 -0
- llm_cheap_filter-0.2.0/pyproject.toml +33 -0
- llm_cheap_filter-0.2.0/setup.cfg +4 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/__init__.py +32 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/analysis.py +227 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/commitments.py +50 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/pipeline.py +310 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/policy.py +68 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/prefilter.py +82 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/py.typed +1 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter/receipt.py +720 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/PKG-INFO +249 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/SOURCES.txt +23 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/dependency_links.txt +1 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/requires.txt +7 -0
- llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/top_level.txt +1 -0
- llm_cheap_filter-0.2.0/tests/test_agent_contract.py +12 -0
- llm_cheap_filter-0.2.0/tests/test_analysis.py +151 -0
- llm_cheap_filter-0.2.0/tests/test_ecosystem_component_contract.py +107 -0
- llm_cheap_filter-0.2.0/tests/test_package_contract.py +26 -0
- llm_cheap_filter-0.2.0/tests/test_pipeline.py +458 -0
- llm_cheap_filter-0.2.0/tests/test_security_portfolio_roadmap_contract.py +35 -0
- llm_cheap_filter-0.2.0/tests/test_triage_receipt.py +251 -0
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 krivonosoff161
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,249 @@
|
|
|
1
|
+
Metadata-Version: 2.4
|
|
2
|
+
Name: llm-cheap-filter
|
|
3
|
+
Version: 0.2.0
|
|
4
|
+
Summary: Deterministic pre-filter + cheap→chief escalation pipeline — send only the few items that matter to your expensive model.
|
|
5
|
+
Author: krivonosoff161
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
Project-URL: Homepage, https://github.com/krivonosoff161/llm-cheap-filter
|
|
8
|
+
Project-URL: Repository, https://github.com/krivonosoff161/llm-cheap-filter
|
|
9
|
+
Keywords: llm,agents,cost,router,pipeline,triage,filter
|
|
10
|
+
Requires-Python: >=3.9
|
|
11
|
+
Description-Content-Type: text/markdown
|
|
12
|
+
License-File: LICENSE
|
|
13
|
+
Provides-Extra: dev
|
|
14
|
+
Requires-Dist: build>=1.2.2; extra == "dev"
|
|
15
|
+
Requires-Dist: pytest>=7; extra == "dev"
|
|
16
|
+
Requires-Dist: ruff>=0.6; extra == "dev"
|
|
17
|
+
Requires-Dist: setuptools>=77; extra == "dev"
|
|
18
|
+
Requires-Dist: wheel>=0.44; extra == "dev"
|
|
19
|
+
Dynamic: license-file
|
|
20
|
+
|
|
21
|
+
# llm-cheap-filter
|
|
22
|
+
|
|
23
|
+
Ecosystem role and current integration status: [component roadmap](docs/component-roadmap.md).
|
|
24
|
+
The public cross-repository plan is owned by the
|
|
25
|
+
[Agentic Security Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md).
|
|
26
|
+
|
|
27
|
+
[](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml)
|
|
28
|
+
[](LICENSE)
|
|
29
|
+
[](https://www.python.org/)
|
|
30
|
+

|
|
31
|
+
|
|
32
|
+
**Don't send every item to your LLM.** Drop obvious noise for free with rules,
|
|
33
|
+
judge the rest with a *cheap* model, and escalate only the few that matter to an
|
|
34
|
+
*expensive* one. A small, **zero-dependency** triage pipeline for agentic systems.
|
|
35
|
+
|
|
36
|
+
> Generalized from a news-triage workflow. The pattern —
|
|
37
|
+
> *deterministic filter → cheap → chief* — is one of the biggest levers on agentic LLM
|
|
38
|
+
> cost, but only if you measure what it drops and escalates.
|
|
39
|
+
|
|
40
|
+
`llm-cheap-filter` is currently a standalone support adapter. Its source tree builds the
|
|
41
|
+
zero-runtime-dependency distribution candidate `llm-cheap-filter==0.2.0`. It is not yet
|
|
42
|
+
published or automatically activated by Harness.
|
|
43
|
+
|
|
44
|
+
---
|
|
45
|
+
|
|
46
|
+
## The idea
|
|
47
|
+
|
|
48
|
+
```
|
|
49
|
+
items in
|
|
50
|
+
│
|
|
51
|
+
┌───────────▼───────────┐
|
|
52
|
+
│ PreFilter (0 tokens) │ drop noise / dupes / too-short by rules
|
|
53
|
+
└───────────┬───────────┘
|
|
54
|
+
survivors │
|
|
55
|
+
┌───────────▼───────────┐
|
|
56
|
+
│ cheap LLM judge │ score + flags (high volume, low price)
|
|
57
|
+
└───────────┬───────────┘
|
|
58
|
+
EscalationPolicy (0 tokens) drop · keep cheap · escalate
|
|
59
|
+
│ few
|
|
60
|
+
┌───────────▼───────────┐
|
|
61
|
+
│ chief LLM decide │ expensive, only for candidates
|
|
62
|
+
└────────────────────────┘
|
|
63
|
+
```
|
|
64
|
+
|
|
65
|
+
The pipeline is **LLM-client-agnostic**: you inject two async callables, so it works with any provider (it pairs naturally with the sibling [`llm-router`](https://github.com/krivonosoff161/llm-router)) or a fake for offline tests.
|
|
66
|
+
|
|
67
|
+
---
|
|
68
|
+
|
|
69
|
+
## Demo (runs offline, no keys)
|
|
70
|
+
|
|
71
|
+
```bash
|
|
72
|
+
python examples/offline_demo.py
|
|
73
|
+
```
|
|
74
|
+
|
|
75
|
+
```text
|
|
76
|
+
[chief ] SEC approves spot ETF — inflows surge score=0.90
|
|
77
|
+
[filtered] Sponsored: trade with XYZ broker score=0.00 noise_match
|
|
78
|
+
[filtered] Weekly recap: what moved markets score=0.00 noise_match
|
|
79
|
+
[chief ] Company files for bankruptcy, halts operations score=0.90
|
|
80
|
+
[filtered] Analyst opinion: why I think it goes up score=0.00 noise_match
|
|
81
|
+
[chief ] Major data breach exposes 10M records score=0.90
|
|
82
|
+
[filtered] Top 5 coins to watch this week score=0.00 noise_match
|
|
83
|
+
[filtered] SEC approves spot ETF — inflows surge score=0.00 duplicate
|
|
84
|
+
[cheap ] Quiet trading day, nothing notable score=0.40
|
|
85
|
+
|
|
86
|
+
summary: {'items_in': 9, 'filtered_free': 5, 'ended_cheap': 1, 'escalated_chief': 3,
|
|
87
|
+
'errors': 0, 'cancelled': 0, 'total_tokens': 228, 'total_cost': 0.0188,
|
|
88
|
+
'chief_rate': 0.333}
|
|
89
|
+
```
|
|
90
|
+
|
|
91
|
+
In this committed synthetic example, 5 of 9 items never touched an LLM and 3
|
|
92
|
+
reached the expensive model. This is example arithmetic, not evidence of a
|
|
93
|
+
production chief rate; real results depend on labels, thresholds, source quality,
|
|
94
|
+
and drift.
|
|
95
|
+
|
|
96
|
+
---
|
|
97
|
+
|
|
98
|
+
## Features
|
|
99
|
+
|
|
100
|
+
- **PreFilter** — pure rules (0 tokens): drop noise substrings, require keep-keywords, min length, near-duplicate dedup (stdlib `difflib`).
|
|
101
|
+
- **EscalationPolicy** — pure rules: `drop` / keep-`cheap` / escalate-`chief` from the cheap stage's score + flags.
|
|
102
|
+
- **Pipeline** — runs the stages, caps concurrency, and returns a per-item report + a cost/savings summary.
|
|
103
|
+
- **Savings report** — estimate actual tokens/cost against an all-chief counterfactual.
|
|
104
|
+
- **Threshold calibration** — sweep cheap-stage thresholds against labeled outcomes and
|
|
105
|
+
measure false accepts / false escalates before tightening.
|
|
106
|
+
- **Bring your own LLM** — inject `cheap_call` / `chief_call`; nothing is hardcoded to a provider.
|
|
107
|
+
- **Zero runtime dependencies** — standard library only. Fully testable offline.
|
|
108
|
+
|
|
109
|
+
---
|
|
110
|
+
|
|
111
|
+
## Install
|
|
112
|
+
|
|
113
|
+
```bash
|
|
114
|
+
git clone https://github.com/krivonosoff161/llm-cheap-filter
|
|
115
|
+
cd llm-cheap-filter
|
|
116
|
+
python -m build
|
|
117
|
+
python -m pip install dist/llm_cheap_filter-0.2.0-py3-none-any.whl
|
|
118
|
+
```
|
|
119
|
+
|
|
120
|
+
For editable development use `python -m pip install -e .[dev]`. Requires **Python 3.9+**.
|
|
121
|
+
CI builds and installs the exact wheel on Ubuntu and Windows. Harness `main` declares a
|
|
122
|
+
source-only `filter` extra, but this package is not on PyPI and published Harness `v1.3.0`
|
|
123
|
+
metadata does not contain that extra. Public
|
|
124
|
+
`pip install agentic-security-harness[filter]` support is therefore unavailable; package
|
|
125
|
+
publication and newer Harness package metadata remain separate release gates. Installing
|
|
126
|
+
the package never calls a provider or activates caller-supplied model functions.
|
|
127
|
+
|
|
128
|
+
---
|
|
129
|
+
|
|
130
|
+
## Quickstart
|
|
131
|
+
|
|
132
|
+
```python
|
|
133
|
+
import asyncio
|
|
134
|
+
from llm_cheap_filter import PreFilter, EscalationPolicy, Pipeline
|
|
135
|
+
|
|
136
|
+
# your LLM, adapted to the expected shapes:
|
|
137
|
+
async def cheap_call(text):
|
|
138
|
+
# -> (judgment with 'score' [+ optional 'flagged'], usage)
|
|
139
|
+
return {"score": 0.8, "flagged": False}, {"total_tokens": 12, "cost_usd": 0.0002}
|
|
140
|
+
|
|
141
|
+
async def chief_call(text, judgment):
|
|
142
|
+
# -> (decision, usage)
|
|
143
|
+
return {"verdict": "ACT"}, {"total_tokens": 60, "cost_usd": 0.006}
|
|
144
|
+
|
|
145
|
+
pipe = Pipeline(
|
|
146
|
+
PreFilter(drop_substrings=("sponsored", "opinion"), min_chars=12, dedup_threshold=90),
|
|
147
|
+
EscalationPolicy(escalate_if_score_at_least=0.65, drop_if_score_below=0.2),
|
|
148
|
+
cheap_call, chief_call,
|
|
149
|
+
)
|
|
150
|
+
|
|
151
|
+
report = asyncio.run(pipe.run(["SEC approves spot ETF", "Sponsored: buy now"]))
|
|
152
|
+
print(report.summary)
|
|
153
|
+
```
|
|
154
|
+
|
|
155
|
+
Pair it with [`llm-router`](https://github.com/krivonosoff161/llm-router) for the real calls — see [examples/with_llm_router.py](examples/with_llm_router.py).
|
|
156
|
+
|
|
157
|
+
### Measure savings
|
|
158
|
+
|
|
159
|
+
```python
|
|
160
|
+
from llm_cheap_filter import build_savings_report
|
|
161
|
+
|
|
162
|
+
savings = build_savings_report(report, chief_tokens_per_item=60, chief_cost_per_item=0.006)
|
|
163
|
+
print(savings.as_dict())
|
|
164
|
+
```
|
|
165
|
+
|
|
166
|
+
### Calibrate thresholds
|
|
167
|
+
|
|
168
|
+
```python
|
|
169
|
+
from llm_cheap_filter import calibrate_thresholds
|
|
170
|
+
|
|
171
|
+
points = calibrate_thresholds(
|
|
172
|
+
scores=[0.95, 0.70, 0.45, 0.20],
|
|
173
|
+
should_escalate=[True, False, True, False],
|
|
174
|
+
thresholds=(0.4, 0.6, 0.8),
|
|
175
|
+
)
|
|
176
|
+
for point in points:
|
|
177
|
+
print(point.as_dict())
|
|
178
|
+
```
|
|
179
|
+
|
|
180
|
+
Use calibration before tightening thresholds. A lower chief rate is not a win if false
|
|
181
|
+
accepts increase.
|
|
182
|
+
|
|
183
|
+
---
|
|
184
|
+
|
|
185
|
+
## How it works
|
|
186
|
+
|
|
187
|
+
**PreFilter** (`prefilter.py`) — per item, in order (so dedup sees prior survivors):
|
|
188
|
+
`drop_substrings` · `keep_keywords` · `min_chars` · `dedup_threshold` (1–100 fuzzy ratio). Returns `keep / score / reason`.
|
|
189
|
+
|
|
190
|
+
**EscalationPolicy** (`policy.py`) — given the cheap score + `flagged`:
|
|
191
|
+
`flagged` or `score ≥ escalate_if_score_at_least` → **chief**; `score < drop_if_score_below` → **drop**; otherwise keep the **cheap** result.
|
|
192
|
+
|
|
193
|
+
**Pipeline** (`pipeline.py`) — prefilter sequentially (free), then run survivors through the LLM stages concurrently (capped by `concurrency`). `report.summary` gives `items_in / filtered_free / ended_cheap / escalated_chief / errors / cancelled / total_tokens / total_cost / chief_rate`.
|
|
194
|
+
|
|
195
|
+
**Analysis helpers** (`analysis.py`) — offline helpers for already-recorded outputs:
|
|
196
|
+
`build_savings_report(report)` estimates actual spend against an all-chief counterfactual,
|
|
197
|
+
and `calibrate_thresholds(scores, should_escalate)` sweeps cheap-stage thresholds to show
|
|
198
|
+
chief rate, false accepts, false escalates, precision, and recall.
|
|
199
|
+
|
|
200
|
+
### Injected callables
|
|
201
|
+
|
|
202
|
+
```text
|
|
203
|
+
cheap_call(text) -> (judgment: dict with 'score' [+ 'flagged'], usage: dict)
|
|
204
|
+
chief_call(text, judgment) -> (decision: dict, usage: dict)
|
|
205
|
+
```
|
|
206
|
+
`usage` may carry `total_tokens` and `cost_usd` (or `cost`); both are tallied. Invalid usage values are reported as per-item `error` results instead of failing the whole batch.
|
|
207
|
+
|
|
208
|
+
---
|
|
209
|
+
|
|
210
|
+
## Tests
|
|
211
|
+
|
|
212
|
+
```bash
|
|
213
|
+
python -m pytest -q # offline, fake LLM, no network
|
|
214
|
+
```
|
|
215
|
+
|
|
216
|
+
---
|
|
217
|
+
|
|
218
|
+
## Docs
|
|
219
|
+
|
|
220
|
+
- [Component roadmap](docs/component-roadmap.md) — source-owned ecosystem role,
|
|
221
|
+
platform evidence, historical projections, and integration gates.
|
|
222
|
+
- [Project map](docs/project-map.md) — modules, what exists today vs not included, reviewer checklist.
|
|
223
|
+
- [Use cases](docs/use-cases.md) — triage, alert fatigue, support, scanning; what this is *not*.
|
|
224
|
+
- [Calibration and replay](docs/calibration-replay.md) — labeled samples, false accepts, false escalates, and report artifacts.
|
|
225
|
+
- [Triage Batch Receipt V1](docs/triage-batch-receipt.md) — canonical digest-only
|
|
226
|
+
batch accounting, explicit loss stages, and authority boundaries.
|
|
227
|
+
- [Examples guide](examples/README.md) — what each example shows and does not prove.
|
|
228
|
+
- [Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md)
|
|
229
|
+
— the canonical public ordering for cross-repository integration work.
|
|
230
|
+
|
|
231
|
+
---
|
|
232
|
+
|
|
233
|
+
## Limitations / non-goals
|
|
234
|
+
|
|
235
|
+
- Text items in, structured judgments out — not a full agent framework.
|
|
236
|
+
- This is a library, not a CLI tool; the scripts in `examples/` are runnable demos.
|
|
237
|
+
- The cheap stage must return a `score`; you own the prompt/parsing (the example shows JSON-mode parsing).
|
|
238
|
+
- A miscalibrated cheap stage can filter out important items. Start with permissive thresholds, replay against labeled samples, and use the per-item reasons before tightening.
|
|
239
|
+
- Dedup uses `difflib` (good for headlines/short text); for very large streams swap in your own near-duplicate check.
|
|
240
|
+
- It controls *which* items reach the expensive model — it does not implement the models themselves.
|
|
241
|
+
- It is not the portfolio flagship, a correctness oracle, or a security control
|
|
242
|
+
by itself. Larger systems own the final decision, validation, authorization,
|
|
243
|
+
storage, and safety boundaries.
|
|
244
|
+
|
|
245
|
+
---
|
|
246
|
+
|
|
247
|
+
## License
|
|
248
|
+
|
|
249
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,229 @@
|
|
|
1
|
+
# llm-cheap-filter
|
|
2
|
+
|
|
3
|
+
Ecosystem role and current integration status: [component roadmap](docs/component-roadmap.md).
|
|
4
|
+
The public cross-repository plan is owned by the
|
|
5
|
+
[Agentic Security Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md).
|
|
6
|
+
|
|
7
|
+
[](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml)
|
|
8
|
+
[](LICENSE)
|
|
9
|
+
[](https://www.python.org/)
|
|
10
|
+

|
|
11
|
+
|
|
12
|
+
**Don't send every item to your LLM.** Drop obvious noise for free with rules,
|
|
13
|
+
judge the rest with a *cheap* model, and escalate only the few that matter to an
|
|
14
|
+
*expensive* one. A small, **zero-dependency** triage pipeline for agentic systems.
|
|
15
|
+
|
|
16
|
+
> Generalized from a news-triage workflow. The pattern —
|
|
17
|
+
> *deterministic filter → cheap → chief* — is one of the biggest levers on agentic LLM
|
|
18
|
+
> cost, but only if you measure what it drops and escalates.
|
|
19
|
+
|
|
20
|
+
`llm-cheap-filter` is currently a standalone support adapter. Its source tree builds the
|
|
21
|
+
zero-runtime-dependency distribution candidate `llm-cheap-filter==0.2.0`. It is not yet
|
|
22
|
+
published or automatically activated by Harness.
|
|
23
|
+
|
|
24
|
+
---
|
|
25
|
+
|
|
26
|
+
## The idea
|
|
27
|
+
|
|
28
|
+
```
|
|
29
|
+
items in
|
|
30
|
+
│
|
|
31
|
+
┌───────────▼───────────┐
|
|
32
|
+
│ PreFilter (0 tokens) │ drop noise / dupes / too-short by rules
|
|
33
|
+
└───────────┬───────────┘
|
|
34
|
+
survivors │
|
|
35
|
+
┌───────────▼───────────┐
|
|
36
|
+
│ cheap LLM judge │ score + flags (high volume, low price)
|
|
37
|
+
└───────────┬───────────┘
|
|
38
|
+
EscalationPolicy (0 tokens) drop · keep cheap · escalate
|
|
39
|
+
│ few
|
|
40
|
+
┌───────────▼───────────┐
|
|
41
|
+
│ chief LLM decide │ expensive, only for candidates
|
|
42
|
+
└────────────────────────┘
|
|
43
|
+
```
|
|
44
|
+
|
|
45
|
+
The pipeline is **LLM-client-agnostic**: you inject two async callables, so it works with any provider (it pairs naturally with the sibling [`llm-router`](https://github.com/krivonosoff161/llm-router)) or a fake for offline tests.
|
|
46
|
+
|
|
47
|
+
---
|
|
48
|
+
|
|
49
|
+
## Demo (runs offline, no keys)
|
|
50
|
+
|
|
51
|
+
```bash
|
|
52
|
+
python examples/offline_demo.py
|
|
53
|
+
```
|
|
54
|
+
|
|
55
|
+
```text
|
|
56
|
+
[chief ] SEC approves spot ETF — inflows surge score=0.90
|
|
57
|
+
[filtered] Sponsored: trade with XYZ broker score=0.00 noise_match
|
|
58
|
+
[filtered] Weekly recap: what moved markets score=0.00 noise_match
|
|
59
|
+
[chief ] Company files for bankruptcy, halts operations score=0.90
|
|
60
|
+
[filtered] Analyst opinion: why I think it goes up score=0.00 noise_match
|
|
61
|
+
[chief ] Major data breach exposes 10M records score=0.90
|
|
62
|
+
[filtered] Top 5 coins to watch this week score=0.00 noise_match
|
|
63
|
+
[filtered] SEC approves spot ETF — inflows surge score=0.00 duplicate
|
|
64
|
+
[cheap ] Quiet trading day, nothing notable score=0.40
|
|
65
|
+
|
|
66
|
+
summary: {'items_in': 9, 'filtered_free': 5, 'ended_cheap': 1, 'escalated_chief': 3,
|
|
67
|
+
'errors': 0, 'cancelled': 0, 'total_tokens': 228, 'total_cost': 0.0188,
|
|
68
|
+
'chief_rate': 0.333}
|
|
69
|
+
```
|
|
70
|
+
|
|
71
|
+
In this committed synthetic example, 5 of 9 items never touched an LLM and 3
|
|
72
|
+
reached the expensive model. This is example arithmetic, not evidence of a
|
|
73
|
+
production chief rate; real results depend on labels, thresholds, source quality,
|
|
74
|
+
and drift.
|
|
75
|
+
|
|
76
|
+
---
|
|
77
|
+
|
|
78
|
+
## Features
|
|
79
|
+
|
|
80
|
+
- **PreFilter** — pure rules (0 tokens): drop noise substrings, require keep-keywords, min length, near-duplicate dedup (stdlib `difflib`).
|
|
81
|
+
- **EscalationPolicy** — pure rules: `drop` / keep-`cheap` / escalate-`chief` from the cheap stage's score + flags.
|
|
82
|
+
- **Pipeline** — runs the stages, caps concurrency, and returns a per-item report + a cost/savings summary.
|
|
83
|
+
- **Savings report** — estimate actual tokens/cost against an all-chief counterfactual.
|
|
84
|
+
- **Threshold calibration** — sweep cheap-stage thresholds against labeled outcomes and
|
|
85
|
+
measure false accepts / false escalates before tightening.
|
|
86
|
+
- **Bring your own LLM** — inject `cheap_call` / `chief_call`; nothing is hardcoded to a provider.
|
|
87
|
+
- **Zero runtime dependencies** — standard library only. Fully testable offline.
|
|
88
|
+
|
|
89
|
+
---
|
|
90
|
+
|
|
91
|
+
## Install
|
|
92
|
+
|
|
93
|
+
```bash
|
|
94
|
+
git clone https://github.com/krivonosoff161/llm-cheap-filter
|
|
95
|
+
cd llm-cheap-filter
|
|
96
|
+
python -m build
|
|
97
|
+
python -m pip install dist/llm_cheap_filter-0.2.0-py3-none-any.whl
|
|
98
|
+
```
|
|
99
|
+
|
|
100
|
+
For editable development use `python -m pip install -e .[dev]`. Requires **Python 3.9+**.
|
|
101
|
+
CI builds and installs the exact wheel on Ubuntu and Windows. Harness `main` declares a
|
|
102
|
+
source-only `filter` extra, but this package is not on PyPI and published Harness `v1.3.0`
|
|
103
|
+
metadata does not contain that extra. Public
|
|
104
|
+
`pip install agentic-security-harness[filter]` support is therefore unavailable; package
|
|
105
|
+
publication and newer Harness package metadata remain separate release gates. Installing
|
|
106
|
+
the package never calls a provider or activates caller-supplied model functions.
|
|
107
|
+
|
|
108
|
+
---
|
|
109
|
+
|
|
110
|
+
## Quickstart
|
|
111
|
+
|
|
112
|
+
```python
|
|
113
|
+
import asyncio
|
|
114
|
+
from llm_cheap_filter import PreFilter, EscalationPolicy, Pipeline
|
|
115
|
+
|
|
116
|
+
# your LLM, adapted to the expected shapes:
|
|
117
|
+
async def cheap_call(text):
|
|
118
|
+
# -> (judgment with 'score' [+ optional 'flagged'], usage)
|
|
119
|
+
return {"score": 0.8, "flagged": False}, {"total_tokens": 12, "cost_usd": 0.0002}
|
|
120
|
+
|
|
121
|
+
async def chief_call(text, judgment):
|
|
122
|
+
# -> (decision, usage)
|
|
123
|
+
return {"verdict": "ACT"}, {"total_tokens": 60, "cost_usd": 0.006}
|
|
124
|
+
|
|
125
|
+
pipe = Pipeline(
|
|
126
|
+
PreFilter(drop_substrings=("sponsored", "opinion"), min_chars=12, dedup_threshold=90),
|
|
127
|
+
EscalationPolicy(escalate_if_score_at_least=0.65, drop_if_score_below=0.2),
|
|
128
|
+
cheap_call, chief_call,
|
|
129
|
+
)
|
|
130
|
+
|
|
131
|
+
report = asyncio.run(pipe.run(["SEC approves spot ETF", "Sponsored: buy now"]))
|
|
132
|
+
print(report.summary)
|
|
133
|
+
```
|
|
134
|
+
|
|
135
|
+
Pair it with [`llm-router`](https://github.com/krivonosoff161/llm-router) for the real calls — see [examples/with_llm_router.py](examples/with_llm_router.py).
|
|
136
|
+
|
|
137
|
+
### Measure savings
|
|
138
|
+
|
|
139
|
+
```python
|
|
140
|
+
from llm_cheap_filter import build_savings_report
|
|
141
|
+
|
|
142
|
+
savings = build_savings_report(report, chief_tokens_per_item=60, chief_cost_per_item=0.006)
|
|
143
|
+
print(savings.as_dict())
|
|
144
|
+
```
|
|
145
|
+
|
|
146
|
+
### Calibrate thresholds
|
|
147
|
+
|
|
148
|
+
```python
|
|
149
|
+
from llm_cheap_filter import calibrate_thresholds
|
|
150
|
+
|
|
151
|
+
points = calibrate_thresholds(
|
|
152
|
+
scores=[0.95, 0.70, 0.45, 0.20],
|
|
153
|
+
should_escalate=[True, False, True, False],
|
|
154
|
+
thresholds=(0.4, 0.6, 0.8),
|
|
155
|
+
)
|
|
156
|
+
for point in points:
|
|
157
|
+
print(point.as_dict())
|
|
158
|
+
```
|
|
159
|
+
|
|
160
|
+
Use calibration before tightening thresholds. A lower chief rate is not a win if false
|
|
161
|
+
accepts increase.
|
|
162
|
+
|
|
163
|
+
---
|
|
164
|
+
|
|
165
|
+
## How it works
|
|
166
|
+
|
|
167
|
+
**PreFilter** (`prefilter.py`) — per item, in order (so dedup sees prior survivors):
|
|
168
|
+
`drop_substrings` · `keep_keywords` · `min_chars` · `dedup_threshold` (1–100 fuzzy ratio). Returns `keep / score / reason`.
|
|
169
|
+
|
|
170
|
+
**EscalationPolicy** (`policy.py`) — given the cheap score + `flagged`:
|
|
171
|
+
`flagged` or `score ≥ escalate_if_score_at_least` → **chief**; `score < drop_if_score_below` → **drop**; otherwise keep the **cheap** result.
|
|
172
|
+
|
|
173
|
+
**Pipeline** (`pipeline.py`) — prefilter sequentially (free), then run survivors through the LLM stages concurrently (capped by `concurrency`). `report.summary` gives `items_in / filtered_free / ended_cheap / escalated_chief / errors / cancelled / total_tokens / total_cost / chief_rate`.
|
|
174
|
+
|
|
175
|
+
**Analysis helpers** (`analysis.py`) — offline helpers for already-recorded outputs:
|
|
176
|
+
`build_savings_report(report)` estimates actual spend against an all-chief counterfactual,
|
|
177
|
+
and `calibrate_thresholds(scores, should_escalate)` sweeps cheap-stage thresholds to show
|
|
178
|
+
chief rate, false accepts, false escalates, precision, and recall.
|
|
179
|
+
|
|
180
|
+
### Injected callables
|
|
181
|
+
|
|
182
|
+
```text
|
|
183
|
+
cheap_call(text) -> (judgment: dict with 'score' [+ 'flagged'], usage: dict)
|
|
184
|
+
chief_call(text, judgment) -> (decision: dict, usage: dict)
|
|
185
|
+
```
|
|
186
|
+
`usage` may carry `total_tokens` and `cost_usd` (or `cost`); both are tallied. Invalid usage values are reported as per-item `error` results instead of failing the whole batch.
|
|
187
|
+
|
|
188
|
+
---
|
|
189
|
+
|
|
190
|
+
## Tests
|
|
191
|
+
|
|
192
|
+
```bash
|
|
193
|
+
python -m pytest -q # offline, fake LLM, no network
|
|
194
|
+
```
|
|
195
|
+
|
|
196
|
+
---
|
|
197
|
+
|
|
198
|
+
## Docs
|
|
199
|
+
|
|
200
|
+
- [Component roadmap](docs/component-roadmap.md) — source-owned ecosystem role,
|
|
201
|
+
platform evidence, historical projections, and integration gates.
|
|
202
|
+
- [Project map](docs/project-map.md) — modules, what exists today vs not included, reviewer checklist.
|
|
203
|
+
- [Use cases](docs/use-cases.md) — triage, alert fatigue, support, scanning; what this is *not*.
|
|
204
|
+
- [Calibration and replay](docs/calibration-replay.md) — labeled samples, false accepts, false escalates, and report artifacts.
|
|
205
|
+
- [Triage Batch Receipt V1](docs/triage-batch-receipt.md) — canonical digest-only
|
|
206
|
+
batch accounting, explicit loss stages, and authority boundaries.
|
|
207
|
+
- [Examples guide](examples/README.md) — what each example shows and does not prove.
|
|
208
|
+
- [Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md)
|
|
209
|
+
— the canonical public ordering for cross-repository integration work.
|
|
210
|
+
|
|
211
|
+
---
|
|
212
|
+
|
|
213
|
+
## Limitations / non-goals
|
|
214
|
+
|
|
215
|
+
- Text items in, structured judgments out — not a full agent framework.
|
|
216
|
+
- This is a library, not a CLI tool; the scripts in `examples/` are runnable demos.
|
|
217
|
+
- The cheap stage must return a `score`; you own the prompt/parsing (the example shows JSON-mode parsing).
|
|
218
|
+
- A miscalibrated cheap stage can filter out important items. Start with permissive thresholds, replay against labeled samples, and use the per-item reasons before tightening.
|
|
219
|
+
- Dedup uses `difflib` (good for headlines/short text); for very large streams swap in your own near-duplicate check.
|
|
220
|
+
- It controls *which* items reach the expensive model — it does not implement the models themselves.
|
|
221
|
+
- It is not the portfolio flagship, a correctness oracle, or a security control
|
|
222
|
+
by itself. Larger systems own the final decision, validation, authorization,
|
|
223
|
+
storage, and safety boundaries.
|
|
224
|
+
|
|
225
|
+
---
|
|
226
|
+
|
|
227
|
+
## License
|
|
228
|
+
|
|
229
|
+
MIT — see [LICENSE](LICENSE).
|
|
@@ -0,0 +1,33 @@
|
|
|
1
|
+
[build-system]
|
|
2
|
+
requires = ["setuptools>=77", "wheel"]
|
|
3
|
+
build-backend = "setuptools.build_meta"
|
|
4
|
+
|
|
5
|
+
[project]
|
|
6
|
+
name = "llm-cheap-filter"
|
|
7
|
+
version = "0.2.0"
|
|
8
|
+
description = "Deterministic pre-filter + cheap→chief escalation pipeline — send only the few items that matter to your expensive model."
|
|
9
|
+
readme = "README.md"
|
|
10
|
+
requires-python = ">=3.9"
|
|
11
|
+
license = "MIT"
|
|
12
|
+
authors = [{ name = "krivonosoff161" }]
|
|
13
|
+
keywords = ["llm", "agents", "cost", "router", "pipeline", "triage", "filter"]
|
|
14
|
+
dependencies = [] # zero runtime dependencies (standard library only)
|
|
15
|
+
|
|
16
|
+
[project.optional-dependencies]
|
|
17
|
+
dev = ["build>=1.2.2", "pytest>=7", "ruff>=0.6", "setuptools>=77", "wheel>=0.44"]
|
|
18
|
+
|
|
19
|
+
[project.urls]
|
|
20
|
+
Homepage = "https://github.com/krivonosoff161/llm-cheap-filter"
|
|
21
|
+
Repository = "https://github.com/krivonosoff161/llm-cheap-filter"
|
|
22
|
+
|
|
23
|
+
[tool.setuptools.packages.find]
|
|
24
|
+
where = ["src"]
|
|
25
|
+
|
|
26
|
+
[tool.setuptools.package-data]
|
|
27
|
+
llm_cheap_filter = ["py.typed"]
|
|
28
|
+
|
|
29
|
+
[tool.pytest.ini_options]
|
|
30
|
+
testpaths = ["tests"]
|
|
31
|
+
|
|
32
|
+
[tool.ruff.lint]
|
|
33
|
+
select = ["E4", "E7", "E9", "F"]
|
|
@@ -0,0 +1,32 @@
|
|
|
1
|
+
# -*- coding: utf-8 -*-
|
|
2
|
+
"""llm-cheap-filter — deterministic pre-filter + cheap→chief escalation pipeline."""
|
|
3
|
+
from .analysis import CalibrationPoint, SavingsReport, build_savings_report, calibrate_thresholds
|
|
4
|
+
from .pipeline import CheapCall, ChiefCall, ItemResult, Pipeline, Report
|
|
5
|
+
from .policy import CHEAP, CHIEF, DROP, EscalationPolicy
|
|
6
|
+
from .prefilter import PreFilter, PreVerdict
|
|
7
|
+
from .receipt import (
|
|
8
|
+
TRIAGE_BATCH_RECEIPT_V1,
|
|
9
|
+
TriageBatchReceiptV1,
|
|
10
|
+
TriageReceiptContractError,
|
|
11
|
+
TriageReceiptResultV1,
|
|
12
|
+
TriageReceiptSummaryV1,
|
|
13
|
+
build_triage_batch_receipt_v1,
|
|
14
|
+
decode_triage_batch_receipt_v1,
|
|
15
|
+
encode_triage_batch_receipt_v1,
|
|
16
|
+
escalation_policy_sha256,
|
|
17
|
+
prefilter_configuration_sha256,
|
|
18
|
+
triage_batch_receipt_v1_json_schema,
|
|
19
|
+
)
|
|
20
|
+
|
|
21
|
+
__all__ = [
|
|
22
|
+
"PreFilter", "PreVerdict",
|
|
23
|
+
"EscalationPolicy", "DROP", "CHEAP", "CHIEF",
|
|
24
|
+
"Pipeline", "ItemResult", "Report", "CheapCall", "ChiefCall",
|
|
25
|
+
"SavingsReport", "CalibrationPoint", "build_savings_report", "calibrate_thresholds",
|
|
26
|
+
"TRIAGE_BATCH_RECEIPT_V1", "TriageBatchReceiptV1", "TriageReceiptResultV1",
|
|
27
|
+
"TriageReceiptSummaryV1", "TriageReceiptContractError",
|
|
28
|
+
"build_triage_batch_receipt_v1", "encode_triage_batch_receipt_v1",
|
|
29
|
+
"decode_triage_batch_receipt_v1", "triage_batch_receipt_v1_json_schema",
|
|
30
|
+
"prefilter_configuration_sha256", "escalation_policy_sha256",
|
|
31
|
+
]
|
|
32
|
+
__version__ = "0.2.0"
|