bounded-distill 0.0.1__tar.gz
This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
- bounded_distill-0.0.1/.gitignore +18 -0
- bounded_distill-0.0.1/LICENSE +21 -0
- bounded_distill-0.0.1/PKG-INFO +108 -0
- bounded_distill-0.0.1/README.md +83 -0
- bounded_distill-0.0.1/pyproject.toml +51 -0
- bounded_distill-0.0.1/src/bounded_distill/__init__.py +8 -0
- bounded_distill-0.0.1/src/bounded_distill/py.typed +0 -0
|
@@ -0,0 +1,18 @@
|
|
|
1
|
+
__pycache__/
|
|
2
|
+
*.py[cod]
|
|
3
|
+
.venv/
|
|
4
|
+
dist/
|
|
5
|
+
build/
|
|
6
|
+
wheels/
|
|
7
|
+
*.egg-info/
|
|
8
|
+
.pytest_cache/
|
|
9
|
+
.ruff_cache/
|
|
10
|
+
.mypy_cache/
|
|
11
|
+
.env
|
|
12
|
+
*.joblib
|
|
13
|
+
*.skops
|
|
14
|
+
data/
|
|
15
|
+
|
|
16
|
+
# Private working notes (not for the public repo or the PyPI sdist)
|
|
17
|
+
ClaudeContext/
|
|
18
|
+
CLAUDE.md
|
|
@@ -0,0 +1,21 @@
|
|
|
1
|
+
MIT License
|
|
2
|
+
|
|
3
|
+
Copyright (c) 2026 Yerassyl Pirzhanov
|
|
4
|
+
|
|
5
|
+
Permission is hereby granted, free of charge, to any person obtaining a copy
|
|
6
|
+
of this software and associated documentation files (the "Software"), to deal
|
|
7
|
+
in the Software without restriction, including without limitation the rights
|
|
8
|
+
to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
|
|
9
|
+
copies of the Software, and to permit persons to whom the Software is
|
|
10
|
+
furnished to do so, subject to the following conditions:
|
|
11
|
+
|
|
12
|
+
The above copyright notice and this permission notice shall be included in all
|
|
13
|
+
copies or substantial portions of the Software.
|
|
14
|
+
|
|
15
|
+
THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
|
|
16
|
+
IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
|
|
17
|
+
FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
|
|
18
|
+
AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
|
|
19
|
+
LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
|
|
20
|
+
OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
|
|
21
|
+
SOFTWARE.
|
|
@@ -0,0 +1,108 @@
|
|
|
1
|
+
Metadata-Version: 2.5
|
|
2
|
+
Name: bounded-distill
|
|
3
|
+
Version: 0.0.1
|
|
4
|
+
Summary: Distill your LLM classifier into a small fast model, with a statistical bound on agreement and LLM fallback for uncertain cases.
|
|
5
|
+
Author-email: Yerassyl Pirzhanov <erasil200270@gmail.com>
|
|
6
|
+
License-Expression: MIT
|
|
7
|
+
License-File: LICENSE
|
|
8
|
+
Keywords: calibration,cascade,confidence-threshold,cost-reduction,distillation,knowledge-distillation,label-noise,llm,text-classification
|
|
9
|
+
Classifier: Development Status :: 1 - Planning
|
|
10
|
+
Classifier: Intended Audience :: Developers
|
|
11
|
+
Classifier: Intended Audience :: Science/Research
|
|
12
|
+
Classifier: Operating System :: OS Independent
|
|
13
|
+
Classifier: Programming Language :: Python :: 3
|
|
14
|
+
Classifier: Programming Language :: Python :: 3 :: Only
|
|
15
|
+
Classifier: Programming Language :: Python :: 3.10
|
|
16
|
+
Classifier: Programming Language :: Python :: 3.11
|
|
17
|
+
Classifier: Programming Language :: Python :: 3.12
|
|
18
|
+
Classifier: Programming Language :: Python :: 3.13
|
|
19
|
+
Classifier: Programming Language :: Python :: 3.14
|
|
20
|
+
Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
|
|
21
|
+
Classifier: Topic :: Text Processing :: Linguistic
|
|
22
|
+
Classifier: Typing :: Typed
|
|
23
|
+
Requires-Python: >=3.10
|
|
24
|
+
Description-Content-Type: text/markdown
|
|
25
|
+
|
|
26
|
+
# bounded-distill
|
|
27
|
+
|
|
28
|
+
**Distill your LLM classifier into a small, fast model — with a statistical bound on agreement.**
|
|
29
|
+
|
|
30
|
+
> **Status: pre-alpha (planning).** Version 0.0.1 contains no working API yet.
|
|
31
|
+
> This README describes the planned design. Feedback is welcome.
|
|
32
|
+
|
|
33
|
+
## The problem
|
|
34
|
+
|
|
35
|
+
Many teams classify text with an LLM pipeline: a prompt, a provider, output parsing.
|
|
36
|
+
It works, but every call is slow and costs money. A common fix is to train a small
|
|
37
|
+
classifier on the LLM's own labels and fall back to the LLM only when the small model
|
|
38
|
+
is unsure.
|
|
39
|
+
|
|
40
|
+
Training that classifier is easy. Deciding **when it is safe to trust it** is the hard
|
|
41
|
+
part. Nobody wants to replace an LLM in production on the basis of "it looked fine on
|
|
42
|
+
a validation set".
|
|
43
|
+
|
|
44
|
+
## What bounded-distill does
|
|
45
|
+
|
|
46
|
+
You give it texts and the labels your LLM already produced (for example, from your
|
|
47
|
+
logs). You state a target, such as *"agree with the LLM at least 97% of the time"*.
|
|
48
|
+
bounded-distill:
|
|
49
|
+
|
|
50
|
+
1. trains a small classifier (frozen sentence embeddings + logistic regression);
|
|
51
|
+
2. picks a confidence threshold so that the **lower confidence bound** of agreement on
|
|
52
|
+
the texts it answers meets your target — not just the point estimate;
|
|
53
|
+
3. reports how much traffic the small model can take over, and the estimated cost and
|
|
54
|
+
latency savings;
|
|
55
|
+
4. lists **suspects**: texts where the small model is confident but disagrees with the
|
|
56
|
+
LLM. These are often LLM labeling mistakes and point to ambiguous class definitions
|
|
57
|
+
in your prompt.
|
|
58
|
+
|
|
59
|
+
Texts below the threshold go back to your own LLM pipeline. The library never calls an
|
|
60
|
+
LLM itself.
|
|
61
|
+
|
|
62
|
+
## Planned API
|
|
63
|
+
|
|
64
|
+
```python
|
|
65
|
+
from bounded_distill import distill
|
|
66
|
+
|
|
67
|
+
router = distill(texts, llm_labels, target_agreement=0.97)
|
|
68
|
+
|
|
69
|
+
label = router.predict(text)
|
|
70
|
+
if label is None: # not confident enough
|
|
71
|
+
label = my_llm_pipeline(text)
|
|
72
|
+
|
|
73
|
+
print(router.report()) # coverage, agreement (with bound), per-class stats, savings
|
|
74
|
+
router.suspects(top_n=50) # confident disagreements worth a human look
|
|
75
|
+
```
|
|
76
|
+
|
|
77
|
+
Optional: pass a small gold-labeled set to see accuracy next to agreement with the LLM.
|
|
78
|
+
|
|
79
|
+
## How it differs from similar tools
|
|
80
|
+
|
|
81
|
+
Several projects distill an LLM classifier into a small model with an LLM fallback.
|
|
82
|
+
bounded-distill focuses on:
|
|
83
|
+
|
|
84
|
+
- **Offline first.** Works on labels you already have; it does not run or wrap the LLM.
|
|
85
|
+
- **A statistical bound**, not a hand-picked threshold.
|
|
86
|
+
- **Auditing the teacher.** Confident disagreements surface LLM labeling errors
|
|
87
|
+
(in the spirit of Confident Learning, Northcutt et al., 2021).
|
|
88
|
+
- **Small and simple.** One function call, strict typing, few dependencies.
|
|
89
|
+
|
|
90
|
+
## Honest limitations
|
|
91
|
+
|
|
92
|
+
- Agreement with the LLM is not accuracy. If the LLM makes the same mistake
|
|
93
|
+
consistently, the small model learns it too. Use a gold set to check.
|
|
94
|
+
- Single-label classification with a fixed, known set of labels only.
|
|
95
|
+
- When the small model answers, you get only the label, not other fields of your
|
|
96
|
+
pipeline's structured output.
|
|
97
|
+
|
|
98
|
+
## Roadmap
|
|
99
|
+
|
|
100
|
+
- [ ] Offline core: `distill`, `Router.predict`, save/load
|
|
101
|
+
- [ ] Report: coverage, agreement with lower bound, per-class metrics, savings
|
|
102
|
+
- [ ] Suspects (confident disagreements)
|
|
103
|
+
- [ ] Benchmarks on public datasets with real LLM labels
|
|
104
|
+
- [ ] Optional online mode (collect pairs from a running pipeline)
|
|
105
|
+
|
|
106
|
+
## License
|
|
107
|
+
|
|
108
|
+
MIT
|
|
@@ -0,0 +1,83 @@
|
|
|
1
|
+
# bounded-distill
|
|
2
|
+
|
|
3
|
+
**Distill your LLM classifier into a small, fast model — with a statistical bound on agreement.**
|
|
4
|
+
|
|
5
|
+
> **Status: pre-alpha (planning).** Version 0.0.1 contains no working API yet.
|
|
6
|
+
> This README describes the planned design. Feedback is welcome.
|
|
7
|
+
|
|
8
|
+
## The problem
|
|
9
|
+
|
|
10
|
+
Many teams classify text with an LLM pipeline: a prompt, a provider, output parsing.
|
|
11
|
+
It works, but every call is slow and costs money. A common fix is to train a small
|
|
12
|
+
classifier on the LLM's own labels and fall back to the LLM only when the small model
|
|
13
|
+
is unsure.
|
|
14
|
+
|
|
15
|
+
Training that classifier is easy. Deciding **when it is safe to trust it** is the hard
|
|
16
|
+
part. Nobody wants to replace an LLM in production on the basis of "it looked fine on
|
|
17
|
+
a validation set".
|
|
18
|
+
|
|
19
|
+
## What bounded-distill does
|
|
20
|
+
|
|
21
|
+
You give it texts and the labels your LLM already produced (for example, from your
|
|
22
|
+
logs). You state a target, such as *"agree with the LLM at least 97% of the time"*.
|
|
23
|
+
bounded-distill:
|
|
24
|
+
|
|
25
|
+
1. trains a small classifier (frozen sentence embeddings + logistic regression);
|
|
26
|
+
2. picks a confidence threshold so that the **lower confidence bound** of agreement on
|
|
27
|
+
the texts it answers meets your target — not just the point estimate;
|
|
28
|
+
3. reports how much traffic the small model can take over, and the estimated cost and
|
|
29
|
+
latency savings;
|
|
30
|
+
4. lists **suspects**: texts where the small model is confident but disagrees with the
|
|
31
|
+
LLM. These are often LLM labeling mistakes and point to ambiguous class definitions
|
|
32
|
+
in your prompt.
|
|
33
|
+
|
|
34
|
+
Texts below the threshold go back to your own LLM pipeline. The library never calls an
|
|
35
|
+
LLM itself.
|
|
36
|
+
|
|
37
|
+
## Planned API
|
|
38
|
+
|
|
39
|
+
```python
|
|
40
|
+
from bounded_distill import distill
|
|
41
|
+
|
|
42
|
+
router = distill(texts, llm_labels, target_agreement=0.97)
|
|
43
|
+
|
|
44
|
+
label = router.predict(text)
|
|
45
|
+
if label is None: # not confident enough
|
|
46
|
+
label = my_llm_pipeline(text)
|
|
47
|
+
|
|
48
|
+
print(router.report()) # coverage, agreement (with bound), per-class stats, savings
|
|
49
|
+
router.suspects(top_n=50) # confident disagreements worth a human look
|
|
50
|
+
```
|
|
51
|
+
|
|
52
|
+
Optional: pass a small gold-labeled set to see accuracy next to agreement with the LLM.
|
|
53
|
+
|
|
54
|
+
## How it differs from similar tools
|
|
55
|
+
|
|
56
|
+
Several projects distill an LLM classifier into a small model with an LLM fallback.
|
|
57
|
+
bounded-distill focuses on:
|
|
58
|
+
|
|
59
|
+
- **Offline first.** Works on labels you already have; it does not run or wrap the LLM.
|
|
60
|
+
- **A statistical bound**, not a hand-picked threshold.
|
|
61
|
+
- **Auditing the teacher.** Confident disagreements surface LLM labeling errors
|
|
62
|
+
(in the spirit of Confident Learning, Northcutt et al., 2021).
|
|
63
|
+
- **Small and simple.** One function call, strict typing, few dependencies.
|
|
64
|
+
|
|
65
|
+
## Honest limitations
|
|
66
|
+
|
|
67
|
+
- Agreement with the LLM is not accuracy. If the LLM makes the same mistake
|
|
68
|
+
consistently, the small model learns it too. Use a gold set to check.
|
|
69
|
+
- Single-label classification with a fixed, known set of labels only.
|
|
70
|
+
- When the small model answers, you get only the label, not other fields of your
|
|
71
|
+
pipeline's structured output.
|
|
72
|
+
|
|
73
|
+
## Roadmap
|
|
74
|
+
|
|
75
|
+
- [ ] Offline core: `distill`, `Router.predict`, save/load
|
|
76
|
+
- [ ] Report: coverage, agreement with lower bound, per-class metrics, savings
|
|
77
|
+
- [ ] Suspects (confident disagreements)
|
|
78
|
+
- [ ] Benchmarks on public datasets with real LLM labels
|
|
79
|
+
- [ ] Optional online mode (collect pairs from a running pipeline)
|
|
80
|
+
|
|
81
|
+
## License
|
|
82
|
+
|
|
83
|
+
MIT
|
|
@@ -0,0 +1,51 @@
|
|
|
1
|
+
[project]
|
|
2
|
+
name = "bounded-distill"
|
|
3
|
+
version = "0.0.1"
|
|
4
|
+
description = "Distill your LLM classifier into a small fast model, with a statistical bound on agreement and LLM fallback for uncertain cases."
|
|
5
|
+
readme = "README.md"
|
|
6
|
+
requires-python = ">=3.10"
|
|
7
|
+
authors = [{ name = "Yerassyl Pirzhanov", email = "erasil200270@gmail.com" }]
|
|
8
|
+
license = "MIT"
|
|
9
|
+
license-files = ["LICENSE"]
|
|
10
|
+
keywords = [
|
|
11
|
+
"llm",
|
|
12
|
+
"distillation",
|
|
13
|
+
"knowledge-distillation",
|
|
14
|
+
"text-classification",
|
|
15
|
+
"cascade",
|
|
16
|
+
"calibration",
|
|
17
|
+
"confidence-threshold",
|
|
18
|
+
"label-noise",
|
|
19
|
+
"cost-reduction",
|
|
20
|
+
]
|
|
21
|
+
classifiers = [
|
|
22
|
+
"Development Status :: 1 - Planning",
|
|
23
|
+
"Intended Audience :: Developers",
|
|
24
|
+
"Intended Audience :: Science/Research",
|
|
25
|
+
"Operating System :: OS Independent",
|
|
26
|
+
"Programming Language :: Python :: 3",
|
|
27
|
+
"Programming Language :: Python :: 3 :: Only",
|
|
28
|
+
"Programming Language :: Python :: 3.10",
|
|
29
|
+
"Programming Language :: Python :: 3.11",
|
|
30
|
+
"Programming Language :: Python :: 3.12",
|
|
31
|
+
"Programming Language :: Python :: 3.13",
|
|
32
|
+
"Programming Language :: Python :: 3.14",
|
|
33
|
+
"Topic :: Scientific/Engineering :: Artificial Intelligence",
|
|
34
|
+
"Topic :: Text Processing :: Linguistic",
|
|
35
|
+
"Typing :: Typed",
|
|
36
|
+
]
|
|
37
|
+
dependencies = []
|
|
38
|
+
|
|
39
|
+
# [project.urls]
|
|
40
|
+
# Homepage = "https://github.com/<your-github>/bounded-distill"
|
|
41
|
+
# Issues = "https://github.com/<your-github>/bounded-distill/issues"
|
|
42
|
+
|
|
43
|
+
[build-system]
|
|
44
|
+
requires = ["hatchling>=1.27"]
|
|
45
|
+
build-backend = "hatchling.build"
|
|
46
|
+
|
|
47
|
+
[tool.hatch.build.targets.wheel]
|
|
48
|
+
packages = ["src/bounded_distill"]
|
|
49
|
+
|
|
50
|
+
[tool.hatch.build.targets.sdist]
|
|
51
|
+
include = ["src/", "README.md", "LICENSE", "pyproject.toml"]
|
|
File without changes
|