llm-cheap-filter 0.2.0__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
Files changed (25) hide show
  1. llm_cheap_filter-0.2.0/LICENSE +21 -0
  2. llm_cheap_filter-0.2.0/PKG-INFO +249 -0
  3. llm_cheap_filter-0.2.0/README.md +229 -0
  4. llm_cheap_filter-0.2.0/pyproject.toml +33 -0
  5. llm_cheap_filter-0.2.0/setup.cfg +4 -0
  6. llm_cheap_filter-0.2.0/src/llm_cheap_filter/__init__.py +32 -0
  7. llm_cheap_filter-0.2.0/src/llm_cheap_filter/analysis.py +227 -0
  8. llm_cheap_filter-0.2.0/src/llm_cheap_filter/commitments.py +50 -0
  9. llm_cheap_filter-0.2.0/src/llm_cheap_filter/pipeline.py +310 -0
  10. llm_cheap_filter-0.2.0/src/llm_cheap_filter/policy.py +68 -0
  11. llm_cheap_filter-0.2.0/src/llm_cheap_filter/prefilter.py +82 -0
  12. llm_cheap_filter-0.2.0/src/llm_cheap_filter/py.typed +1 -0
  13. llm_cheap_filter-0.2.0/src/llm_cheap_filter/receipt.py +720 -0
  14. llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/PKG-INFO +249 -0
  15. llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/SOURCES.txt +23 -0
  16. llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/dependency_links.txt +1 -0
  17. llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/requires.txt +7 -0
  18. llm_cheap_filter-0.2.0/src/llm_cheap_filter.egg-info/top_level.txt +1 -0
  19. llm_cheap_filter-0.2.0/tests/test_agent_contract.py +12 -0
  20. llm_cheap_filter-0.2.0/tests/test_analysis.py +151 -0
  21. llm_cheap_filter-0.2.0/tests/test_ecosystem_component_contract.py +107 -0
  22. llm_cheap_filter-0.2.0/tests/test_package_contract.py +26 -0
  23. llm_cheap_filter-0.2.0/tests/test_pipeline.py +458 -0
  24. llm_cheap_filter-0.2.0/tests/test_security_portfolio_roadmap_contract.py +35 -0
  25. llm_cheap_filter-0.2.0/tests/test_triage_receipt.py +251 -0
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 krivonosoff161
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,249 @@
1
+ Metadata-Version: 2.4
2
+ Name: llm-cheap-filter
3
+ Version: 0.2.0
4
+ Summary: Deterministic pre-filter + cheap→chief escalation pipeline — send only the few items that matter to your expensive model.
5
+ Author: krivonosoff161
6
+ License-Expression: MIT
7
+ Project-URL: Homepage, https://github.com/krivonosoff161/llm-cheap-filter
8
+ Project-URL: Repository, https://github.com/krivonosoff161/llm-cheap-filter
9
+ Keywords: llm,agents,cost,router,pipeline,triage,filter
10
+ Requires-Python: >=3.9
11
+ Description-Content-Type: text/markdown
12
+ License-File: LICENSE
13
+ Provides-Extra: dev
14
+ Requires-Dist: build>=1.2.2; extra == "dev"
15
+ Requires-Dist: pytest>=7; extra == "dev"
16
+ Requires-Dist: ruff>=0.6; extra == "dev"
17
+ Requires-Dist: setuptools>=77; extra == "dev"
18
+ Requires-Dist: wheel>=0.44; extra == "dev"
19
+ Dynamic: license-file
20
+
21
+ # llm-cheap-filter
22
+
23
+ Ecosystem role and current integration status: [component roadmap](docs/component-roadmap.md).
24
+ The public cross-repository plan is owned by the
25
+ [Agentic Security Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md).
26
+
27
+ [![Tests](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml/badge.svg)](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml)
28
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
29
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
30
+ ![deps: none](https://img.shields.io/badge/runtime%20deps-0-brightgreen.svg)
31
+
32
+ **Don't send every item to your LLM.** Drop obvious noise for free with rules,
33
+ judge the rest with a *cheap* model, and escalate only the few that matter to an
34
+ *expensive* one. A small, **zero-dependency** triage pipeline for agentic systems.
35
+
36
+ > Generalized from a news-triage workflow. The pattern —
37
+ > *deterministic filter → cheap → chief* — is one of the biggest levers on agentic LLM
38
+ > cost, but only if you measure what it drops and escalates.
39
+
40
+ `llm-cheap-filter` is currently a standalone support adapter. Its source tree builds the
41
+ zero-runtime-dependency distribution candidate `llm-cheap-filter==0.2.0`. It is not yet
42
+ published or automatically activated by Harness.
43
+
44
+ ---
45
+
46
+ ## The idea
47
+
48
+ ```
49
+ items in
50
+
51
+ ┌───────────▼───────────┐
52
+ │ PreFilter (0 tokens) │ drop noise / dupes / too-short by rules
53
+ └───────────┬───────────┘
54
+ survivors │
55
+ ┌───────────▼───────────┐
56
+ │ cheap LLM judge │ score + flags (high volume, low price)
57
+ └───────────┬───────────┘
58
+ EscalationPolicy (0 tokens) drop · keep cheap · escalate
59
+ │ few
60
+ ┌───────────▼───────────┐
61
+ │ chief LLM decide │ expensive, only for candidates
62
+ └────────────────────────┘
63
+ ```
64
+
65
+ The pipeline is **LLM-client-agnostic**: you inject two async callables, so it works with any provider (it pairs naturally with the sibling [`llm-router`](https://github.com/krivonosoff161/llm-router)) or a fake for offline tests.
66
+
67
+ ---
68
+
69
+ ## Demo (runs offline, no keys)
70
+
71
+ ```bash
72
+ python examples/offline_demo.py
73
+ ```
74
+
75
+ ```text
76
+ [chief ] SEC approves spot ETF — inflows surge score=0.90
77
+ [filtered] Sponsored: trade with XYZ broker score=0.00 noise_match
78
+ [filtered] Weekly recap: what moved markets score=0.00 noise_match
79
+ [chief ] Company files for bankruptcy, halts operations score=0.90
80
+ [filtered] Analyst opinion: why I think it goes up score=0.00 noise_match
81
+ [chief ] Major data breach exposes 10M records score=0.90
82
+ [filtered] Top 5 coins to watch this week score=0.00 noise_match
83
+ [filtered] SEC approves spot ETF — inflows surge score=0.00 duplicate
84
+ [cheap ] Quiet trading day, nothing notable score=0.40
85
+
86
+ summary: {'items_in': 9, 'filtered_free': 5, 'ended_cheap': 1, 'escalated_chief': 3,
87
+ 'errors': 0, 'cancelled': 0, 'total_tokens': 228, 'total_cost': 0.0188,
88
+ 'chief_rate': 0.333}
89
+ ```
90
+
91
+ In this committed synthetic example, 5 of 9 items never touched an LLM and 3
92
+ reached the expensive model. This is example arithmetic, not evidence of a
93
+ production chief rate; real results depend on labels, thresholds, source quality,
94
+ and drift.
95
+
96
+ ---
97
+
98
+ ## Features
99
+
100
+ - **PreFilter** — pure rules (0 tokens): drop noise substrings, require keep-keywords, min length, near-duplicate dedup (stdlib `difflib`).
101
+ - **EscalationPolicy** — pure rules: `drop` / keep-`cheap` / escalate-`chief` from the cheap stage's score + flags.
102
+ - **Pipeline** — runs the stages, caps concurrency, and returns a per-item report + a cost/savings summary.
103
+ - **Savings report** — estimate actual tokens/cost against an all-chief counterfactual.
104
+ - **Threshold calibration** — sweep cheap-stage thresholds against labeled outcomes and
105
+ measure false accepts / false escalates before tightening.
106
+ - **Bring your own LLM** — inject `cheap_call` / `chief_call`; nothing is hardcoded to a provider.
107
+ - **Zero runtime dependencies** — standard library only. Fully testable offline.
108
+
109
+ ---
110
+
111
+ ## Install
112
+
113
+ ```bash
114
+ git clone https://github.com/krivonosoff161/llm-cheap-filter
115
+ cd llm-cheap-filter
116
+ python -m build
117
+ python -m pip install dist/llm_cheap_filter-0.2.0-py3-none-any.whl
118
+ ```
119
+
120
+ For editable development use `python -m pip install -e .[dev]`. Requires **Python 3.9+**.
121
+ CI builds and installs the exact wheel on Ubuntu and Windows. Harness `main` declares a
122
+ source-only `filter` extra, but this package is not on PyPI and published Harness `v1.3.0`
123
+ metadata does not contain that extra. Public
124
+ `pip install agentic-security-harness[filter]` support is therefore unavailable; package
125
+ publication and newer Harness package metadata remain separate release gates. Installing
126
+ the package never calls a provider or activates caller-supplied model functions.
127
+
128
+ ---
129
+
130
+ ## Quickstart
131
+
132
+ ```python
133
+ import asyncio
134
+ from llm_cheap_filter import PreFilter, EscalationPolicy, Pipeline
135
+
136
+ # your LLM, adapted to the expected shapes:
137
+ async def cheap_call(text):
138
+ # -> (judgment with 'score' [+ optional 'flagged'], usage)
139
+ return {"score": 0.8, "flagged": False}, {"total_tokens": 12, "cost_usd": 0.0002}
140
+
141
+ async def chief_call(text, judgment):
142
+ # -> (decision, usage)
143
+ return {"verdict": "ACT"}, {"total_tokens": 60, "cost_usd": 0.006}
144
+
145
+ pipe = Pipeline(
146
+ PreFilter(drop_substrings=("sponsored", "opinion"), min_chars=12, dedup_threshold=90),
147
+ EscalationPolicy(escalate_if_score_at_least=0.65, drop_if_score_below=0.2),
148
+ cheap_call, chief_call,
149
+ )
150
+
151
+ report = asyncio.run(pipe.run(["SEC approves spot ETF", "Sponsored: buy now"]))
152
+ print(report.summary)
153
+ ```
154
+
155
+ Pair it with [`llm-router`](https://github.com/krivonosoff161/llm-router) for the real calls — see [examples/with_llm_router.py](examples/with_llm_router.py).
156
+
157
+ ### Measure savings
158
+
159
+ ```python
160
+ from llm_cheap_filter import build_savings_report
161
+
162
+ savings = build_savings_report(report, chief_tokens_per_item=60, chief_cost_per_item=0.006)
163
+ print(savings.as_dict())
164
+ ```
165
+
166
+ ### Calibrate thresholds
167
+
168
+ ```python
169
+ from llm_cheap_filter import calibrate_thresholds
170
+
171
+ points = calibrate_thresholds(
172
+ scores=[0.95, 0.70, 0.45, 0.20],
173
+ should_escalate=[True, False, True, False],
174
+ thresholds=(0.4, 0.6, 0.8),
175
+ )
176
+ for point in points:
177
+ print(point.as_dict())
178
+ ```
179
+
180
+ Use calibration before tightening thresholds. A lower chief rate is not a win if false
181
+ accepts increase.
182
+
183
+ ---
184
+
185
+ ## How it works
186
+
187
+ **PreFilter** (`prefilter.py`) — per item, in order (so dedup sees prior survivors):
188
+ `drop_substrings` · `keep_keywords` · `min_chars` · `dedup_threshold` (1–100 fuzzy ratio). Returns `keep / score / reason`.
189
+
190
+ **EscalationPolicy** (`policy.py`) — given the cheap score + `flagged`:
191
+ `flagged` or `score ≥ escalate_if_score_at_least` → **chief**; `score < drop_if_score_below` → **drop**; otherwise keep the **cheap** result.
192
+
193
+ **Pipeline** (`pipeline.py`) — prefilter sequentially (free), then run survivors through the LLM stages concurrently (capped by `concurrency`). `report.summary` gives `items_in / filtered_free / ended_cheap / escalated_chief / errors / cancelled / total_tokens / total_cost / chief_rate`.
194
+
195
+ **Analysis helpers** (`analysis.py`) — offline helpers for already-recorded outputs:
196
+ `build_savings_report(report)` estimates actual spend against an all-chief counterfactual,
197
+ and `calibrate_thresholds(scores, should_escalate)` sweeps cheap-stage thresholds to show
198
+ chief rate, false accepts, false escalates, precision, and recall.
199
+
200
+ ### Injected callables
201
+
202
+ ```text
203
+ cheap_call(text) -> (judgment: dict with 'score' [+ 'flagged'], usage: dict)
204
+ chief_call(text, judgment) -> (decision: dict, usage: dict)
205
+ ```
206
+ `usage` may carry `total_tokens` and `cost_usd` (or `cost`); both are tallied. Invalid usage values are reported as per-item `error` results instead of failing the whole batch.
207
+
208
+ ---
209
+
210
+ ## Tests
211
+
212
+ ```bash
213
+ python -m pytest -q # offline, fake LLM, no network
214
+ ```
215
+
216
+ ---
217
+
218
+ ## Docs
219
+
220
+ - [Component roadmap](docs/component-roadmap.md) — source-owned ecosystem role,
221
+ platform evidence, historical projections, and integration gates.
222
+ - [Project map](docs/project-map.md) — modules, what exists today vs not included, reviewer checklist.
223
+ - [Use cases](docs/use-cases.md) — triage, alert fatigue, support, scanning; what this is *not*.
224
+ - [Calibration and replay](docs/calibration-replay.md) — labeled samples, false accepts, false escalates, and report artifacts.
225
+ - [Triage Batch Receipt V1](docs/triage-batch-receipt.md) — canonical digest-only
226
+ batch accounting, explicit loss stages, and authority boundaries.
227
+ - [Examples guide](examples/README.md) — what each example shows and does not prove.
228
+ - [Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md)
229
+ — the canonical public ordering for cross-repository integration work.
230
+
231
+ ---
232
+
233
+ ## Limitations / non-goals
234
+
235
+ - Text items in, structured judgments out — not a full agent framework.
236
+ - This is a library, not a CLI tool; the scripts in `examples/` are runnable demos.
237
+ - The cheap stage must return a `score`; you own the prompt/parsing (the example shows JSON-mode parsing).
238
+ - A miscalibrated cheap stage can filter out important items. Start with permissive thresholds, replay against labeled samples, and use the per-item reasons before tightening.
239
+ - Dedup uses `difflib` (good for headlines/short text); for very large streams swap in your own near-duplicate check.
240
+ - It controls *which* items reach the expensive model — it does not implement the models themselves.
241
+ - It is not the portfolio flagship, a correctness oracle, or a security control
242
+ by itself. Larger systems own the final decision, validation, authorization,
243
+ storage, and safety boundaries.
244
+
245
+ ---
246
+
247
+ ## License
248
+
249
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,229 @@
1
+ # llm-cheap-filter
2
+
3
+ Ecosystem role and current integration status: [component roadmap](docs/component-roadmap.md).
4
+ The public cross-repository plan is owned by the
5
+ [Agentic Security Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md).
6
+
7
+ [![Tests](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml/badge.svg)](https://github.com/krivonosoff161/llm-cheap-filter/actions/workflows/tests.yml)
8
+ [![License: MIT](https://img.shields.io/badge/License-MIT-yellow.svg)](LICENSE)
9
+ [![Python 3.9+](https://img.shields.io/badge/python-3.9%2B-blue.svg)](https://www.python.org/)
10
+ ![deps: none](https://img.shields.io/badge/runtime%20deps-0-brightgreen.svg)
11
+
12
+ **Don't send every item to your LLM.** Drop obvious noise for free with rules,
13
+ judge the rest with a *cheap* model, and escalate only the few that matter to an
14
+ *expensive* one. A small, **zero-dependency** triage pipeline for agentic systems.
15
+
16
+ > Generalized from a news-triage workflow. The pattern —
17
+ > *deterministic filter → cheap → chief* — is one of the biggest levers on agentic LLM
18
+ > cost, but only if you measure what it drops and escalates.
19
+
20
+ `llm-cheap-filter` is currently a standalone support adapter. Its source tree builds the
21
+ zero-runtime-dependency distribution candidate `llm-cheap-filter==0.2.0`. It is not yet
22
+ published or automatically activated by Harness.
23
+
24
+ ---
25
+
26
+ ## The idea
27
+
28
+ ```
29
+ items in
30
+
31
+ ┌───────────▼───────────┐
32
+ │ PreFilter (0 tokens) │ drop noise / dupes / too-short by rules
33
+ └───────────┬───────────┘
34
+ survivors │
35
+ ┌───────────▼───────────┐
36
+ │ cheap LLM judge │ score + flags (high volume, low price)
37
+ └───────────┬───────────┘
38
+ EscalationPolicy (0 tokens) drop · keep cheap · escalate
39
+ │ few
40
+ ┌───────────▼───────────┐
41
+ │ chief LLM decide │ expensive, only for candidates
42
+ └────────────────────────┘
43
+ ```
44
+
45
+ The pipeline is **LLM-client-agnostic**: you inject two async callables, so it works with any provider (it pairs naturally with the sibling [`llm-router`](https://github.com/krivonosoff161/llm-router)) or a fake for offline tests.
46
+
47
+ ---
48
+
49
+ ## Demo (runs offline, no keys)
50
+
51
+ ```bash
52
+ python examples/offline_demo.py
53
+ ```
54
+
55
+ ```text
56
+ [chief ] SEC approves spot ETF — inflows surge score=0.90
57
+ [filtered] Sponsored: trade with XYZ broker score=0.00 noise_match
58
+ [filtered] Weekly recap: what moved markets score=0.00 noise_match
59
+ [chief ] Company files for bankruptcy, halts operations score=0.90
60
+ [filtered] Analyst opinion: why I think it goes up score=0.00 noise_match
61
+ [chief ] Major data breach exposes 10M records score=0.90
62
+ [filtered] Top 5 coins to watch this week score=0.00 noise_match
63
+ [filtered] SEC approves spot ETF — inflows surge score=0.00 duplicate
64
+ [cheap ] Quiet trading day, nothing notable score=0.40
65
+
66
+ summary: {'items_in': 9, 'filtered_free': 5, 'ended_cheap': 1, 'escalated_chief': 3,
67
+ 'errors': 0, 'cancelled': 0, 'total_tokens': 228, 'total_cost': 0.0188,
68
+ 'chief_rate': 0.333}
69
+ ```
70
+
71
+ In this committed synthetic example, 5 of 9 items never touched an LLM and 3
72
+ reached the expensive model. This is example arithmetic, not evidence of a
73
+ production chief rate; real results depend on labels, thresholds, source quality,
74
+ and drift.
75
+
76
+ ---
77
+
78
+ ## Features
79
+
80
+ - **PreFilter** — pure rules (0 tokens): drop noise substrings, require keep-keywords, min length, near-duplicate dedup (stdlib `difflib`).
81
+ - **EscalationPolicy** — pure rules: `drop` / keep-`cheap` / escalate-`chief` from the cheap stage's score + flags.
82
+ - **Pipeline** — runs the stages, caps concurrency, and returns a per-item report + a cost/savings summary.
83
+ - **Savings report** — estimate actual tokens/cost against an all-chief counterfactual.
84
+ - **Threshold calibration** — sweep cheap-stage thresholds against labeled outcomes and
85
+ measure false accepts / false escalates before tightening.
86
+ - **Bring your own LLM** — inject `cheap_call` / `chief_call`; nothing is hardcoded to a provider.
87
+ - **Zero runtime dependencies** — standard library only. Fully testable offline.
88
+
89
+ ---
90
+
91
+ ## Install
92
+
93
+ ```bash
94
+ git clone https://github.com/krivonosoff161/llm-cheap-filter
95
+ cd llm-cheap-filter
96
+ python -m build
97
+ python -m pip install dist/llm_cheap_filter-0.2.0-py3-none-any.whl
98
+ ```
99
+
100
+ For editable development use `python -m pip install -e .[dev]`. Requires **Python 3.9+**.
101
+ CI builds and installs the exact wheel on Ubuntu and Windows. Harness `main` declares a
102
+ source-only `filter` extra, but this package is not on PyPI and published Harness `v1.3.0`
103
+ metadata does not contain that extra. Public
104
+ `pip install agentic-security-harness[filter]` support is therefore unavailable; package
105
+ publication and newer Harness package metadata remain separate release gates. Installing
106
+ the package never calls a provider or activates caller-supplied model functions.
107
+
108
+ ---
109
+
110
+ ## Quickstart
111
+
112
+ ```python
113
+ import asyncio
114
+ from llm_cheap_filter import PreFilter, EscalationPolicy, Pipeline
115
+
116
+ # your LLM, adapted to the expected shapes:
117
+ async def cheap_call(text):
118
+ # -> (judgment with 'score' [+ optional 'flagged'], usage)
119
+ return {"score": 0.8, "flagged": False}, {"total_tokens": 12, "cost_usd": 0.0002}
120
+
121
+ async def chief_call(text, judgment):
122
+ # -> (decision, usage)
123
+ return {"verdict": "ACT"}, {"total_tokens": 60, "cost_usd": 0.006}
124
+
125
+ pipe = Pipeline(
126
+ PreFilter(drop_substrings=("sponsored", "opinion"), min_chars=12, dedup_threshold=90),
127
+ EscalationPolicy(escalate_if_score_at_least=0.65, drop_if_score_below=0.2),
128
+ cheap_call, chief_call,
129
+ )
130
+
131
+ report = asyncio.run(pipe.run(["SEC approves spot ETF", "Sponsored: buy now"]))
132
+ print(report.summary)
133
+ ```
134
+
135
+ Pair it with [`llm-router`](https://github.com/krivonosoff161/llm-router) for the real calls — see [examples/with_llm_router.py](examples/with_llm_router.py).
136
+
137
+ ### Measure savings
138
+
139
+ ```python
140
+ from llm_cheap_filter import build_savings_report
141
+
142
+ savings = build_savings_report(report, chief_tokens_per_item=60, chief_cost_per_item=0.006)
143
+ print(savings.as_dict())
144
+ ```
145
+
146
+ ### Calibrate thresholds
147
+
148
+ ```python
149
+ from llm_cheap_filter import calibrate_thresholds
150
+
151
+ points = calibrate_thresholds(
152
+ scores=[0.95, 0.70, 0.45, 0.20],
153
+ should_escalate=[True, False, True, False],
154
+ thresholds=(0.4, 0.6, 0.8),
155
+ )
156
+ for point in points:
157
+ print(point.as_dict())
158
+ ```
159
+
160
+ Use calibration before tightening thresholds. A lower chief rate is not a win if false
161
+ accepts increase.
162
+
163
+ ---
164
+
165
+ ## How it works
166
+
167
+ **PreFilter** (`prefilter.py`) — per item, in order (so dedup sees prior survivors):
168
+ `drop_substrings` · `keep_keywords` · `min_chars` · `dedup_threshold` (1–100 fuzzy ratio). Returns `keep / score / reason`.
169
+
170
+ **EscalationPolicy** (`policy.py`) — given the cheap score + `flagged`:
171
+ `flagged` or `score ≥ escalate_if_score_at_least` → **chief**; `score < drop_if_score_below` → **drop**; otherwise keep the **cheap** result.
172
+
173
+ **Pipeline** (`pipeline.py`) — prefilter sequentially (free), then run survivors through the LLM stages concurrently (capped by `concurrency`). `report.summary` gives `items_in / filtered_free / ended_cheap / escalated_chief / errors / cancelled / total_tokens / total_cost / chief_rate`.
174
+
175
+ **Analysis helpers** (`analysis.py`) — offline helpers for already-recorded outputs:
176
+ `build_savings_report(report)` estimates actual spend against an all-chief counterfactual,
177
+ and `calibrate_thresholds(scores, should_escalate)` sweeps cheap-stage thresholds to show
178
+ chief rate, false accepts, false escalates, precision, and recall.
179
+
180
+ ### Injected callables
181
+
182
+ ```text
183
+ cheap_call(text) -> (judgment: dict with 'score' [+ 'flagged'], usage: dict)
184
+ chief_call(text, judgment) -> (decision: dict, usage: dict)
185
+ ```
186
+ `usage` may carry `total_tokens` and `cost_usd` (or `cost`); both are tallied. Invalid usage values are reported as per-item `error` results instead of failing the whole batch.
187
+
188
+ ---
189
+
190
+ ## Tests
191
+
192
+ ```bash
193
+ python -m pytest -q # offline, fake LLM, no network
194
+ ```
195
+
196
+ ---
197
+
198
+ ## Docs
199
+
200
+ - [Component roadmap](docs/component-roadmap.md) — source-owned ecosystem role,
201
+ platform evidence, historical projections, and integration gates.
202
+ - [Project map](docs/project-map.md) — modules, what exists today vs not included, reviewer checklist.
203
+ - [Use cases](docs/use-cases.md) — triage, alert fatigue, support, scanning; what this is *not*.
204
+ - [Calibration and replay](docs/calibration-replay.md) — labeled samples, false accepts, false escalates, and report artifacts.
205
+ - [Triage Batch Receipt V1](docs/triage-batch-receipt.md) — canonical digest-only
206
+ batch accounting, explicit loss stages, and authority boundaries.
207
+ - [Examples guide](examples/README.md) — what each example shows and does not prove.
208
+ - [Harness ecosystem roadmap](https://github.com/krivonosoff161/agentic-security-harness/blob/main/docs/ecosystem-roadmap.md)
209
+ — the canonical public ordering for cross-repository integration work.
210
+
211
+ ---
212
+
213
+ ## Limitations / non-goals
214
+
215
+ - Text items in, structured judgments out — not a full agent framework.
216
+ - This is a library, not a CLI tool; the scripts in `examples/` are runnable demos.
217
+ - The cheap stage must return a `score`; you own the prompt/parsing (the example shows JSON-mode parsing).
218
+ - A miscalibrated cheap stage can filter out important items. Start with permissive thresholds, replay against labeled samples, and use the per-item reasons before tightening.
219
+ - Dedup uses `difflib` (good for headlines/short text); for very large streams swap in your own near-duplicate check.
220
+ - It controls *which* items reach the expensive model — it does not implement the models themselves.
221
+ - It is not the portfolio flagship, a correctness oracle, or a security control
222
+ by itself. Larger systems own the final decision, validation, authorization,
223
+ storage, and safety boundaries.
224
+
225
+ ---
226
+
227
+ ## License
228
+
229
+ MIT — see [LICENSE](LICENSE).
@@ -0,0 +1,33 @@
1
+ [build-system]
2
+ requires = ["setuptools>=77", "wheel"]
3
+ build-backend = "setuptools.build_meta"
4
+
5
+ [project]
6
+ name = "llm-cheap-filter"
7
+ version = "0.2.0"
8
+ description = "Deterministic pre-filter + cheap→chief escalation pipeline — send only the few items that matter to your expensive model."
9
+ readme = "README.md"
10
+ requires-python = ">=3.9"
11
+ license = "MIT"
12
+ authors = [{ name = "krivonosoff161" }]
13
+ keywords = ["llm", "agents", "cost", "router", "pipeline", "triage", "filter"]
14
+ dependencies = [] # zero runtime dependencies (standard library only)
15
+
16
+ [project.optional-dependencies]
17
+ dev = ["build>=1.2.2", "pytest>=7", "ruff>=0.6", "setuptools>=77", "wheel>=0.44"]
18
+
19
+ [project.urls]
20
+ Homepage = "https://github.com/krivonosoff161/llm-cheap-filter"
21
+ Repository = "https://github.com/krivonosoff161/llm-cheap-filter"
22
+
23
+ [tool.setuptools.packages.find]
24
+ where = ["src"]
25
+
26
+ [tool.setuptools.package-data]
27
+ llm_cheap_filter = ["py.typed"]
28
+
29
+ [tool.pytest.ini_options]
30
+ testpaths = ["tests"]
31
+
32
+ [tool.ruff.lint]
33
+ select = ["E4", "E7", "E9", "F"]
@@ -0,0 +1,4 @@
1
+ [egg_info]
2
+ tag_build =
3
+ tag_date = 0
4
+
@@ -0,0 +1,32 @@
1
+ # -*- coding: utf-8 -*-
2
+ """llm-cheap-filter — deterministic pre-filter + cheap→chief escalation pipeline."""
3
+ from .analysis import CalibrationPoint, SavingsReport, build_savings_report, calibrate_thresholds
4
+ from .pipeline import CheapCall, ChiefCall, ItemResult, Pipeline, Report
5
+ from .policy import CHEAP, CHIEF, DROP, EscalationPolicy
6
+ from .prefilter import PreFilter, PreVerdict
7
+ from .receipt import (
8
+ TRIAGE_BATCH_RECEIPT_V1,
9
+ TriageBatchReceiptV1,
10
+ TriageReceiptContractError,
11
+ TriageReceiptResultV1,
12
+ TriageReceiptSummaryV1,
13
+ build_triage_batch_receipt_v1,
14
+ decode_triage_batch_receipt_v1,
15
+ encode_triage_batch_receipt_v1,
16
+ escalation_policy_sha256,
17
+ prefilter_configuration_sha256,
18
+ triage_batch_receipt_v1_json_schema,
19
+ )
20
+
21
+ __all__ = [
22
+ "PreFilter", "PreVerdict",
23
+ "EscalationPolicy", "DROP", "CHEAP", "CHIEF",
24
+ "Pipeline", "ItemResult", "Report", "CheapCall", "ChiefCall",
25
+ "SavingsReport", "CalibrationPoint", "build_savings_report", "calibrate_thresholds",
26
+ "TRIAGE_BATCH_RECEIPT_V1", "TriageBatchReceiptV1", "TriageReceiptResultV1",
27
+ "TriageReceiptSummaryV1", "TriageReceiptContractError",
28
+ "build_triage_batch_receipt_v1", "encode_triage_batch_receipt_v1",
29
+ "decode_triage_batch_receipt_v1", "triage_batch_receipt_v1_json_schema",
30
+ "prefilter_configuration_sha256", "escalation_policy_sha256",
31
+ ]
32
+ __version__ = "0.2.0"