bounded-distill 0.0.1__tar.gz

This diff represents the content of publicly available package versions that have been released to one of the supported registries. The information contained in this diff is provided for informational purposes only and reflects changes between package versions as they appear in their respective public registries.
@@ -0,0 +1,18 @@
1
+ __pycache__/
2
+ *.py[cod]
3
+ .venv/
4
+ dist/
5
+ build/
6
+ wheels/
7
+ *.egg-info/
8
+ .pytest_cache/
9
+ .ruff_cache/
10
+ .mypy_cache/
11
+ .env
12
+ *.joblib
13
+ *.skops
14
+ data/
15
+
16
+ # Private working notes (not for the public repo or the PyPI sdist)
17
+ ClaudeContext/
18
+ CLAUDE.md
@@ -0,0 +1,21 @@
1
+ MIT License
2
+
3
+ Copyright (c) 2026 Yerassyl Pirzhanov
4
+
5
+ Permission is hereby granted, free of charge, to any person obtaining a copy
6
+ of this software and associated documentation files (the "Software"), to deal
7
+ in the Software without restriction, including without limitation the rights
8
+ to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
9
+ copies of the Software, and to permit persons to whom the Software is
10
+ furnished to do so, subject to the following conditions:
11
+
12
+ The above copyright notice and this permission notice shall be included in all
13
+ copies or substantial portions of the Software.
14
+
15
+ THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
16
+ IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
17
+ FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
18
+ AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
19
+ LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
20
+ OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
21
+ SOFTWARE.
@@ -0,0 +1,108 @@
1
+ Metadata-Version: 2.5
2
+ Name: bounded-distill
3
+ Version: 0.0.1
4
+ Summary: Distill your LLM classifier into a small fast model, with a statistical bound on agreement and LLM fallback for uncertain cases.
5
+ Author-email: Yerassyl Pirzhanov <erasil200270@gmail.com>
6
+ License-Expression: MIT
7
+ License-File: LICENSE
8
+ Keywords: calibration,cascade,confidence-threshold,cost-reduction,distillation,knowledge-distillation,label-noise,llm,text-classification
9
+ Classifier: Development Status :: 1 - Planning
10
+ Classifier: Intended Audience :: Developers
11
+ Classifier: Intended Audience :: Science/Research
12
+ Classifier: Operating System :: OS Independent
13
+ Classifier: Programming Language :: Python :: 3
14
+ Classifier: Programming Language :: Python :: 3 :: Only
15
+ Classifier: Programming Language :: Python :: 3.10
16
+ Classifier: Programming Language :: Python :: 3.11
17
+ Classifier: Programming Language :: Python :: 3.12
18
+ Classifier: Programming Language :: Python :: 3.13
19
+ Classifier: Programming Language :: Python :: 3.14
20
+ Classifier: Topic :: Scientific/Engineering :: Artificial Intelligence
21
+ Classifier: Topic :: Text Processing :: Linguistic
22
+ Classifier: Typing :: Typed
23
+ Requires-Python: >=3.10
24
+ Description-Content-Type: text/markdown
25
+
26
+ # bounded-distill
27
+
28
+ **Distill your LLM classifier into a small, fast model — with a statistical bound on agreement.**
29
+
30
+ > **Status: pre-alpha (planning).** Version 0.0.1 contains no working API yet.
31
+ > This README describes the planned design. Feedback is welcome.
32
+
33
+ ## The problem
34
+
35
+ Many teams classify text with an LLM pipeline: a prompt, a provider, output parsing.
36
+ It works, but every call is slow and costs money. A common fix is to train a small
37
+ classifier on the LLM's own labels and fall back to the LLM only when the small model
38
+ is unsure.
39
+
40
+ Training that classifier is easy. Deciding **when it is safe to trust it** is the hard
41
+ part. Nobody wants to replace an LLM in production on the basis of "it looked fine on
42
+ a validation set".
43
+
44
+ ## What bounded-distill does
45
+
46
+ You give it texts and the labels your LLM already produced (for example, from your
47
+ logs). You state a target, such as *"agree with the LLM at least 97% of the time"*.
48
+ bounded-distill:
49
+
50
+ 1. trains a small classifier (frozen sentence embeddings + logistic regression);
51
+ 2. picks a confidence threshold so that the **lower confidence bound** of agreement on
52
+ the texts it answers meets your target — not just the point estimate;
53
+ 3. reports how much traffic the small model can take over, and the estimated cost and
54
+ latency savings;
55
+ 4. lists **suspects**: texts where the small model is confident but disagrees with the
56
+ LLM. These are often LLM labeling mistakes and point to ambiguous class definitions
57
+ in your prompt.
58
+
59
+ Texts below the threshold go back to your own LLM pipeline. The library never calls an
60
+ LLM itself.
61
+
62
+ ## Planned API
63
+
64
+ ```python
65
+ from bounded_distill import distill
66
+
67
+ router = distill(texts, llm_labels, target_agreement=0.97)
68
+
69
+ label = router.predict(text)
70
+ if label is None: # not confident enough
71
+ label = my_llm_pipeline(text)
72
+
73
+ print(router.report()) # coverage, agreement (with bound), per-class stats, savings
74
+ router.suspects(top_n=50) # confident disagreements worth a human look
75
+ ```
76
+
77
+ Optional: pass a small gold-labeled set to see accuracy next to agreement with the LLM.
78
+
79
+ ## How it differs from similar tools
80
+
81
+ Several projects distill an LLM classifier into a small model with an LLM fallback.
82
+ bounded-distill focuses on:
83
+
84
+ - **Offline first.** Works on labels you already have; it does not run or wrap the LLM.
85
+ - **A statistical bound**, not a hand-picked threshold.
86
+ - **Auditing the teacher.** Confident disagreements surface LLM labeling errors
87
+ (in the spirit of Confident Learning, Northcutt et al., 2021).
88
+ - **Small and simple.** One function call, strict typing, few dependencies.
89
+
90
+ ## Honest limitations
91
+
92
+ - Agreement with the LLM is not accuracy. If the LLM makes the same mistake
93
+ consistently, the small model learns it too. Use a gold set to check.
94
+ - Single-label classification with a fixed, known set of labels only.
95
+ - When the small model answers, you get only the label, not other fields of your
96
+ pipeline's structured output.
97
+
98
+ ## Roadmap
99
+
100
+ - [ ] Offline core: `distill`, `Router.predict`, save/load
101
+ - [ ] Report: coverage, agreement with lower bound, per-class metrics, savings
102
+ - [ ] Suspects (confident disagreements)
103
+ - [ ] Benchmarks on public datasets with real LLM labels
104
+ - [ ] Optional online mode (collect pairs from a running pipeline)
105
+
106
+ ## License
107
+
108
+ MIT
@@ -0,0 +1,83 @@
1
+ # bounded-distill
2
+
3
+ **Distill your LLM classifier into a small, fast model — with a statistical bound on agreement.**
4
+
5
+ > **Status: pre-alpha (planning).** Version 0.0.1 contains no working API yet.
6
+ > This README describes the planned design. Feedback is welcome.
7
+
8
+ ## The problem
9
+
10
+ Many teams classify text with an LLM pipeline: a prompt, a provider, output parsing.
11
+ It works, but every call is slow and costs money. A common fix is to train a small
12
+ classifier on the LLM's own labels and fall back to the LLM only when the small model
13
+ is unsure.
14
+
15
+ Training that classifier is easy. Deciding **when it is safe to trust it** is the hard
16
+ part. Nobody wants to replace an LLM in production on the basis of "it looked fine on
17
+ a validation set".
18
+
19
+ ## What bounded-distill does
20
+
21
+ You give it texts and the labels your LLM already produced (for example, from your
22
+ logs). You state a target, such as *"agree with the LLM at least 97% of the time"*.
23
+ bounded-distill:
24
+
25
+ 1. trains a small classifier (frozen sentence embeddings + logistic regression);
26
+ 2. picks a confidence threshold so that the **lower confidence bound** of agreement on
27
+ the texts it answers meets your target — not just the point estimate;
28
+ 3. reports how much traffic the small model can take over, and the estimated cost and
29
+ latency savings;
30
+ 4. lists **suspects**: texts where the small model is confident but disagrees with the
31
+ LLM. These are often LLM labeling mistakes and point to ambiguous class definitions
32
+ in your prompt.
33
+
34
+ Texts below the threshold go back to your own LLM pipeline. The library never calls an
35
+ LLM itself.
36
+
37
+ ## Planned API
38
+
39
+ ```python
40
+ from bounded_distill import distill
41
+
42
+ router = distill(texts, llm_labels, target_agreement=0.97)
43
+
44
+ label = router.predict(text)
45
+ if label is None: # not confident enough
46
+ label = my_llm_pipeline(text)
47
+
48
+ print(router.report()) # coverage, agreement (with bound), per-class stats, savings
49
+ router.suspects(top_n=50) # confident disagreements worth a human look
50
+ ```
51
+
52
+ Optional: pass a small gold-labeled set to see accuracy next to agreement with the LLM.
53
+
54
+ ## How it differs from similar tools
55
+
56
+ Several projects distill an LLM classifier into a small model with an LLM fallback.
57
+ bounded-distill focuses on:
58
+
59
+ - **Offline first.** Works on labels you already have; it does not run or wrap the LLM.
60
+ - **A statistical bound**, not a hand-picked threshold.
61
+ - **Auditing the teacher.** Confident disagreements surface LLM labeling errors
62
+ (in the spirit of Confident Learning, Northcutt et al., 2021).
63
+ - **Small and simple.** One function call, strict typing, few dependencies.
64
+
65
+ ## Honest limitations
66
+
67
+ - Agreement with the LLM is not accuracy. If the LLM makes the same mistake
68
+ consistently, the small model learns it too. Use a gold set to check.
69
+ - Single-label classification with a fixed, known set of labels only.
70
+ - When the small model answers, you get only the label, not other fields of your
71
+ pipeline's structured output.
72
+
73
+ ## Roadmap
74
+
75
+ - [ ] Offline core: `distill`, `Router.predict`, save/load
76
+ - [ ] Report: coverage, agreement with lower bound, per-class metrics, savings
77
+ - [ ] Suspects (confident disagreements)
78
+ - [ ] Benchmarks on public datasets with real LLM labels
79
+ - [ ] Optional online mode (collect pairs from a running pipeline)
80
+
81
+ ## License
82
+
83
+ MIT
@@ -0,0 +1,51 @@
1
+ [project]
2
+ name = "bounded-distill"
3
+ version = "0.0.1"
4
+ description = "Distill your LLM classifier into a small fast model, with a statistical bound on agreement and LLM fallback for uncertain cases."
5
+ readme = "README.md"
6
+ requires-python = ">=3.10"
7
+ authors = [{ name = "Yerassyl Pirzhanov", email = "erasil200270@gmail.com" }]
8
+ license = "MIT"
9
+ license-files = ["LICENSE"]
10
+ keywords = [
11
+ "llm",
12
+ "distillation",
13
+ "knowledge-distillation",
14
+ "text-classification",
15
+ "cascade",
16
+ "calibration",
17
+ "confidence-threshold",
18
+ "label-noise",
19
+ "cost-reduction",
20
+ ]
21
+ classifiers = [
22
+ "Development Status :: 1 - Planning",
23
+ "Intended Audience :: Developers",
24
+ "Intended Audience :: Science/Research",
25
+ "Operating System :: OS Independent",
26
+ "Programming Language :: Python :: 3",
27
+ "Programming Language :: Python :: 3 :: Only",
28
+ "Programming Language :: Python :: 3.10",
29
+ "Programming Language :: Python :: 3.11",
30
+ "Programming Language :: Python :: 3.12",
31
+ "Programming Language :: Python :: 3.13",
32
+ "Programming Language :: Python :: 3.14",
33
+ "Topic :: Scientific/Engineering :: Artificial Intelligence",
34
+ "Topic :: Text Processing :: Linguistic",
35
+ "Typing :: Typed",
36
+ ]
37
+ dependencies = []
38
+
39
+ # [project.urls]
40
+ # Homepage = "https://github.com/<your-github>/bounded-distill"
41
+ # Issues = "https://github.com/<your-github>/bounded-distill/issues"
42
+
43
+ [build-system]
44
+ requires = ["hatchling>=1.27"]
45
+ build-backend = "hatchling.build"
46
+
47
+ [tool.hatch.build.targets.wheel]
48
+ packages = ["src/bounded_distill"]
49
+
50
+ [tool.hatch.build.targets.sdist]
51
+ include = ["src/", "README.md", "LICENSE", "pyproject.toml"]
@@ -0,0 +1,8 @@
1
+ """bounded-distill: distill an LLM classifier into a small model with a statistical bound on agreement.
2
+
3
+ Pre-alpha. The public API is not implemented yet; see the README for the planned design.
4
+ """
5
+
6
+ __version__ = "0.0.1"
7
+
8
+ __all__ = ["__version__"]
File without changes