tinyconformal 0.1.0__tar.gz

tinyconformal-0.1.0/LICENSE

@@ -0,0 +1,21 @@
+MIT License
+
+Copyright (c) 2025 Lucas Leão
+
+Permission is hereby granted, free of charge, to any person obtaining a copy
+of this software and associated documentation files (the "Software"), to deal
+in the Software without restriction, including without limitation the rights
+to use, copy, modify, merge, publish, distribute, sublicense, and/or sell
+copies of the Software, and to permit persons to whom the Software is
+furnished to do so, subject to the following conditions:
+
+The above copyright notice and this permission notice shall be included in all
+copies or substantial portions of the Software.
+
+THE SOFTWARE IS PROVIDED "AS IS", WITHOUT WARRANTY OF ANY KIND, EXPRESS OR
+IMPLIED, INCLUDING BUT NOT LIMITED TO THE WARRANTIES OF MERCHANTABILITY,
+FITNESS FOR A PARTICULAR PURPOSE AND NONINFRINGEMENT. IN NO EVENT SHALL THE
+AUTHORS OR COPYRIGHT HOLDERS BE LIABLE FOR ANY CLAIM, DAMAGES OR OTHER
+LIABILITY, WHETHER IN AN ACTION OF CONTRACT, TORT OR OTHERWISE, ARISING FROM,
+OUT OF OR IN CONNECTION WITH THE SOFTWARE OR THE USE OR OTHER DEALINGS IN THE
+SOFTWARE.

tinyconformal-0.1.0/PKG-INFO

@@ -0,0 +1,176 @@
+Metadata-Version: 2.4
+Name: tinyconformal
+Version: 0.1.0
+Summary: A small toolbox for conformal prediction
+Author-email: Lucas Leão <heylucasleao@gmail.com>
+License-Expression: MIT
+Keywords: machine-learning,conformal-prediction
+Classifier: Programming Language :: Python :: 3
+Requires-Python: >=3.10
+Description-Content-Type: text/markdown
+License-File: LICENSE
+Requires-Dist: numpy>=1.24.4
+Requires-Dist: venn-abers>=1.4.6
+Requires-Dist: scikit-learn>=1.3.0
+Requires-Dist: quantile-forest>=1.4.2
+Provides-Extra: plot
+Requires-Dist: plotly>=5.22.0; extra == "plot"
+Requires-Dist: kaleido<=0.2.1; extra == "plot"
+Provides-Extra: notebook
+Requires-Dist: nbformat>=5.10.4; extra == "notebook"
+Requires-Dist: ipykernel>=6.7.0; extra == "notebook"
+Provides-Extra: all
+Requires-Dist: plot; extra == "all"
+Requires-Dist: notebook; extra == "all"
+Dynamic: license-file
+
+# TinyConformal
+TinyConformal is an experimental Python library for conformal predictions, providing tools to generate valid prediction sets with a specified significance level (alpha). This project aims to facilitate the implementation of personal and future projects on the topic.
+
+For more information on a previous project related to Out-of-Bag (OOB) solutions, visit [this link](https://github.com/HeyLucasLeao/cp-study).
+
+## Recent updates
+- Added support for exactness-bound-based calibration through `ExactnessBound` for ICP and CQR workflows.
+- Added `unlabeled_fit` support for conformal classifiers and regressors, enabling calibration without labeled calibration data when an exactness bound is available.
+- Classifiers can now be calibrated from unlabeled data using pseudo-labels derived from model predictions, while regressors can use a pre-estimated exactness bound to build the conformity scores.
+
+Previously, `calibrate` used `Balanced Accuracy Score`; it can now also be calibrated with `Matthews Correlation Coefficient` or `Bookmaker Informedness Score` for improved reliability. The `evaluate` method also reports `bm` and `mcc`.
+
+Currently, TinyConformal supports Out-of-Bag (OOB) solutions for `RandomForestClassifier` in binary classification problems, as well as `RandomForestRegressor` and `RandomForestQuantileRegressor` for regression tasks. For additional options and advanced features, you may want to explore [Crepes](https://github.com/henrikbostrom/crepes).
+
+## Installation
+
+Install TinyConformal using pip:
+
+```bash
+pip install tinyconformal
+```
+
+> **Note:** If you want to enable plotting capabilities, you need to install the extras using Poetry:
+
+```bash
+poetry install --E plot
+```
+
+## Usage
+
+### Importing Classifiers
+
+Import the conformal classifiers from the `tinyconformal.classifier` module:
+
+```python
+from tinyconformal.classifier import BinaryClassConditionalConformalClassifier
+from tinyconformal.classifier import BinaryMarginalConformalClassifier
+```
+### Importing Regressors
+
+Import the conformal regressors from the `tinyconformal.regressor` module:
+
+```python
+from tinyconformal.regressor import ConformalizedRegressor
+from tinyconformal.regressor import ConformalizedQuantileRegressor
+```
+### Example
+
+Example usage of `BinaryClassConditionalConformalClassifier`:
+
+```python
+from sklearn.ensemble import RandomForestClassifier
+from tinyconformal.classifier import BinaryClassConditionalConformalClassifier
+
+# Create and fit a RandomForestClassifier
+learner = RandomForestClassifier(n_estimators=100, oob_score=True)
+X_train, y_train = ...  # your training data
+learner.fit(X_train, y_train)
+
+# Create and fit the conformal classifier
+conformal_classifier = BinaryClassConditionalConformalClassifier(learner)
+conformal_classifier.fit(X=X_train, y=y_train, oob=True)
+
+# Make predictions
+X_test = ...  # your test data
+predictions = conformal_classifier.predict(X_test)
+```
+
+### Unlabeled calibration example
+
+For settings where labeled calibration data are unavailable, you can fit the conformal model directly on unlabeled data:
+
+```python
+from sklearn.ensemble import RandomForestClassifier
+from tinyconformal.classifier import BinaryMarginalConformalClassifier
+
+learner = RandomForestClassifier(n_estimators=100, oob_score=True)
+learner.fit(X_train, y_train)
+
+conformal_classifier = BinaryMarginalConformalClassifier(learner)
+conformal_classifier.unlabeled_fit(X_unlabeled)
+
+predictions = conformal_classifier.predict(X_test)
+```
+
+For regressors, you can combine an exactness bound estimate with unlabeled calibration:
+
+```python
+from sklearn.ensemble import RandomForestRegressor
+from tinyconformal import ConformalizedRegressor, ExactnessBound
+
+learner = RandomForestRegressor(random_state=42)
+tilde_beta = ExactnessBound.estimate_icp_bound(learner, X_train, y_train, p=0.95, cv=5)
+
+regressor = ConformalizedRegressor(learner, alpha=0.05)
+regressor.unlabeled_fit(X_unlabeled, tilde_beta=tilde_beta)
+
+intervals = regressor.predict_interval(X_test)
+```
+
+### Evaluating the Classifier
+
+Evaluate the performance of the conformal classifier using the `evaluate` method:
+
+```python
+results = conformal_classifier.evaluate(X_test, y_test)
+print(results)
+```
+
+## Classes
+
+### BinaryMarginalConformalClassifier
+
+`BinaryMarginalConformalClassifier` is a marginal-coverage conformal classifier that uses a classifier as the underlying learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X)`
+
+### BinaryClassConditionalConformalClassifier
+
+`BinaryClassConditionalConformalClassifier` is a class-conditional conformal classifier that uses a classifier as the underlying learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X)` using pseudo-labels derived from the model probabilities
+
+### ConformalizedRegressor
+
+`ConformalizedRegressor` is a conformal regressor built on a regression learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X, tilde_beta=...)` using an exactness bound
+
+### ConformalizedQuantileRegressor
+
+`ConformalizedQuantileRegressor` is a conformal quantile regressor built on a quantile regressor.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X, tilde_beta=...)` using an exactness bound
+
+### ExactnessBound
+
+`ExactnessBound` provides helper methods to estimate the exactness bound used in unlabeled conformal calibration for ICP and CQR workflows.
+
+## License
+
+This project is licensed under the MIT License.

tinyconformal-0.1.0/README.md

@@ -0,0 +1,150 @@
+# TinyConformal
+TinyConformal is an experimental Python library for conformal predictions, providing tools to generate valid prediction sets with a specified significance level (alpha). This project aims to facilitate the implementation of personal and future projects on the topic.
+
+For more information on a previous project related to Out-of-Bag (OOB) solutions, visit [this link](https://github.com/HeyLucasLeao/cp-study).
+
+## Recent updates
+- Added support for exactness-bound-based calibration through `ExactnessBound` for ICP and CQR workflows.
+- Added `unlabeled_fit` support for conformal classifiers and regressors, enabling calibration without labeled calibration data when an exactness bound is available.
+- Classifiers can now be calibrated from unlabeled data using pseudo-labels derived from model predictions, while regressors can use a pre-estimated exactness bound to build the conformity scores.
+
+Previously, `calibrate` used `Balanced Accuracy Score`; it can now also be calibrated with `Matthews Correlation Coefficient` or `Bookmaker Informedness Score` for improved reliability. The `evaluate` method also reports `bm` and `mcc`.
+
+Currently, TinyConformal supports Out-of-Bag (OOB) solutions for `RandomForestClassifier` in binary classification problems, as well as `RandomForestRegressor` and `RandomForestQuantileRegressor` for regression tasks. For additional options and advanced features, you may want to explore [Crepes](https://github.com/henrikbostrom/crepes).
+
+## Installation
+
+Install TinyConformal using pip:
+
+```bash
+pip install tinyconformal
+```
+
+> **Note:** If you want to enable plotting capabilities, you need to install the extras using Poetry:
+
+```bash
+poetry install --E plot
+```
+
+## Usage
+
+### Importing Classifiers
+
+Import the conformal classifiers from the `tinyconformal.classifier` module:
+
+```python
+from tinyconformal.classifier import BinaryClassConditionalConformalClassifier
+from tinyconformal.classifier import BinaryMarginalConformalClassifier
+```
+### Importing Regressors
+
+Import the conformal regressors from the `tinyconformal.regressor` module:
+
+```python
+from tinyconformal.regressor import ConformalizedRegressor
+from tinyconformal.regressor import ConformalizedQuantileRegressor
+```
+### Example
+
+Example usage of `BinaryClassConditionalConformalClassifier`:
+
+```python
+from sklearn.ensemble import RandomForestClassifier
+from tinyconformal.classifier import BinaryClassConditionalConformalClassifier
+
+# Create and fit a RandomForestClassifier
+learner = RandomForestClassifier(n_estimators=100, oob_score=True)
+X_train, y_train = ...  # your training data
+learner.fit(X_train, y_train)
+
+# Create and fit the conformal classifier
+conformal_classifier = BinaryClassConditionalConformalClassifier(learner)
+conformal_classifier.fit(X=X_train, y=y_train, oob=True)
+
+# Make predictions
+X_test = ...  # your test data
+predictions = conformal_classifier.predict(X_test)
+```
+
+### Unlabeled calibration example
+
+For settings where labeled calibration data are unavailable, you can fit the conformal model directly on unlabeled data:
+
+```python
+from sklearn.ensemble import RandomForestClassifier
+from tinyconformal.classifier import BinaryMarginalConformalClassifier
+
+learner = RandomForestClassifier(n_estimators=100, oob_score=True)
+learner.fit(X_train, y_train)
+
+conformal_classifier = BinaryMarginalConformalClassifier(learner)
+conformal_classifier.unlabeled_fit(X_unlabeled)
+
+predictions = conformal_classifier.predict(X_test)
+```
+
+For regressors, you can combine an exactness bound estimate with unlabeled calibration:
+
+```python
+from sklearn.ensemble import RandomForestRegressor
+from tinyconformal import ConformalizedRegressor, ExactnessBound
+
+learner = RandomForestRegressor(random_state=42)
+tilde_beta = ExactnessBound.estimate_icp_bound(learner, X_train, y_train, p=0.95, cv=5)
+
+regressor = ConformalizedRegressor(learner, alpha=0.05)
+regressor.unlabeled_fit(X_unlabeled, tilde_beta=tilde_beta)
+
+intervals = regressor.predict_interval(X_test)
+```
+
+### Evaluating the Classifier
+
+Evaluate the performance of the conformal classifier using the `evaluate` method:
+
+```python
+results = conformal_classifier.evaluate(X_test, y_test)
+print(results)
+```
+
+## Classes
+
+### BinaryMarginalConformalClassifier
+
+`BinaryMarginalConformalClassifier` is a marginal-coverage conformal classifier that uses a classifier as the underlying learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X)`
+
+### BinaryClassConditionalConformalClassifier
+
+`BinaryClassConditionalConformalClassifier` is a class-conditional conformal classifier that uses a classifier as the underlying learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X)` using pseudo-labels derived from the model probabilities
+
+### ConformalizedRegressor
+
+`ConformalizedRegressor` is a conformal regressor built on a regression learner.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X, tilde_beta=...)` using an exactness bound
+
+### ConformalizedQuantileRegressor
+
+`ConformalizedQuantileRegressor` is a conformal quantile regressor built on a quantile regressor.
+
+- Training via labeled calibration: `fit(X, y)`
+- Training via OOB calibration: `fit(X, y, oob=True)`
+- Training via unlabeled calibration: `unlabeled_fit(X, tilde_beta=...)` using an exactness bound
+
+### ExactnessBound
+
+`ExactnessBound` provides helper methods to estimate the exactness bound used in unlabeled conformal calibration for ICP and CQR workflows.
+
+## License
+
+This project is licensed under the MIT License.

tinyconformal-0.1.0/pyproject.toml

@@ -0,0 +1,23 @@
+[project]
+name = "tinyconformal"
+version = "0.1.0"
+description = "A small toolbox for conformal prediction"
+license = "MIT"
+authors = [{name = "Lucas Leão", email = "heylucasleao@gmail.com"}]
+keywords = ["machine-learning", "conformal-prediction"]
+readme = "README.md"
+classifiers = [
+    "Programming Language :: Python :: 3",
+]
+dependencies = [
+    "numpy (>=1.24.4)",
+    "venn-abers (>=1.4.6)",
+    "scikit-learn (>=1.3.0)",
+    "quantile-forest>=1.4.2",
+]
+requires-python = ">=3.10"
+
+[project.optional-dependencies]
+plot = ["plotly (>=5.22.0)", "kaleido (<=0.2.1)"]
+notebook = ["nbformat (>=5.10.4)", "ipykernel (>=6.7.0)"]
+all = ["plot", "notebook"]

tinyconformal-0.1.0/setup.cfg

@@ -0,0 +1,4 @@
+[egg_info]
+tag_build = 
+tag_date = 0
+

tinyconformal-0.1.0/tests/test_classifiers.py

@@ -0,0 +1,144 @@
+# Copyright (c) 2024-2026 Lucas Leão
+# tinyCP - A small toolbox for conformal prediction
+# Licensed under the MIT License
+
+
+import unittest
+import numpy as np
+from sklearn.ensemble import RandomForestClassifier
+from sklearn.model_selection import train_test_split
+from sklearn.datasets import make_classification
+from tinyconformal.classifier.marginal import BinaryMarginalConformalClassifier
+from tinyconformal.classifier.class_conditional import (
+    BinaryClassConditionalConformalClassifier,
+)
+
+
+class TestClassifiers(unittest.TestCase):
+    def setUp(self):
+        weights = [0.4, 0.6]
+        seed = 42
+
+        X, y = make_classification(
+            n_samples=1500,
+            n_features=20,
+            n_informative=2,
+            weights=weights,
+            random_state=seed,
+            n_redundant=2,
+        )
+
+        X_train, X_test, y_train, y_test = train_test_split(
+            X, y, test_size=0.2, random_state=seed, stratify=y
+        )
+        X_train, X_calib, y_train, y_calib = train_test_split(
+            X_train, y_train, test_size=0.25, random_state=seed, stratify=y_train
+        )
+
+        self.X_train = X_train
+        self.y_train = y_train
+
+        self.X_calib = X_calib
+        self.y_calib = y_calib
+
+        self.X_test = X_test
+        self.y_test = y_test
+
+        self.learner = RandomForestClassifier(oob_score=True, n_estimators=10)
+        self.learner.fit(self.X_train, self.y_train)
+
+    def test_marginal_classifier(self):
+        classifier = BinaryMarginalConformalClassifier(self.learner)
+        classifier.fit(self.X_calib, self.y_calib, oob=False)
+
+        classifier.calibrate(self.X_calib, self.y_calib)
+        self.assertTrue(0 < classifier.alpha <= 0.2)
+
+        y_proba = classifier.predict_proba(self.X_test)
+        self.assertEqual(y_proba.shape, (self.X_test.shape[0], 2))
+
+        prediction_set = classifier.predict_set(self.X_test)
+        self.assertEqual(prediction_set.shape, (self.X_test.shape[0], 2))
+
+        p_values = classifier.predict_p(self.X_test)
+        self.assertEqual(p_values.shape, (self.X_test.shape[0], 2))
+
+        y_pred = classifier.predict(self.X_test)
+        self.assertEqual(y_pred.shape, (self.X_test.shape[0],))
+
+        eval_dict = classifier.evaluate(self.X_test, self.y_test)
+        self.assertTrue(isinstance(eval_dict, dict))
+        self.assertEqual(len(eval_dict.keys()), 13)
+
+    def test_class_cond_classifier(self):
+        classifier = BinaryClassConditionalConformalClassifier(self.learner)
+        classifier.fit(self.X_calib, self.y_calib, oob=False)
+
+        classifier.calibrate(self.X_calib, self.y_calib)
+        self.assertTrue(0 < classifier.alpha <= 0.2)
+
+        y_proba = classifier.predict_proba(self.X_test)
+        self.assertEqual(y_proba.shape, (self.X_test.shape[0], 2))
+
+        prediction_set = classifier.predict_set(self.X_test)
+        self.assertEqual(prediction_set.shape, (self.X_test.shape[0], 2))
+
+        p_values = classifier.predict_p(self.X_test)
+        self.assertEqual(p_values.shape, (self.X_test.shape[0], 2))
+
+        y_pred = classifier.predict(self.X_test)
+        self.assertEqual(y_pred.shape, (self.X_test.shape[0],))
+
+        eval_dict = classifier.evaluate(self.X_test, self.y_test)
+        self.assertTrue(isinstance(eval_dict, dict))
+        self.assertEqual(len(eval_dict.keys()), 13)
+
+    def test_oob_marginal_classifier(self):
+        classifier = BinaryMarginalConformalClassifier(self.learner)
+        classifier.fit(y=self.y_train, oob=True)
+
+        classifier.calibrate(self.X_calib, self.y_calib)
+        self.assertTrue(0 < classifier.alpha <= 0.2)
+
+        y_proba = classifier.predict_proba(self.X_test)
+        self.assertEqual(y_proba.shape, (self.X_test.shape[0], 2))
+
+        prediction_set = classifier.predict_set(self.X_test)
+        self.assertEqual(prediction_set.shape, (self.X_test.shape[0], 2))
+
+        p_values = classifier.predict_p(self.X_test)
+        self.assertEqual(p_values.shape, (self.X_test.shape[0], 2))
+
+        y_pred = classifier.predict(self.X_test)
+        self.assertEqual(y_pred.shape, (self.X_test.shape[0],))
+
+        eval_dict = classifier.evaluate(self.X_test, self.y_test)
+        self.assertTrue(isinstance(eval_dict, dict))
+        self.assertEqual(len(eval_dict.keys()), 13)
+
+    def test_oob_class_conditional_classifier(self):
+        classifier = BinaryClassConditionalConformalClassifier(self.learner)
+        classifier.fit(y=self.y_train, oob=True)
+
+        classifier.calibrate(self.X_calib, self.y_calib)
+        self.assertTrue(0 < classifier.alpha <= 0.2)
+
+        y_proba = classifier.predict_proba(self.X_test)
+        self.assertEqual(y_proba.shape, (self.X_test.shape[0], 2))
+
+        prediction_set = classifier.predict_set(self.X_test)
+        self.assertEqual(prediction_set.shape, (self.X_test.shape[0], 2))
+
+        p_values = classifier.predict_p(self.X_test)
+        self.assertEqual(p_values.shape, (self.X_test.shape[0], 2))
+
+        y_pred = classifier.predict(self.X_test)
+        self.assertEqual(y_pred.shape, (self.X_test.shape[0],))
+
+        eval_dict = classifier.evaluate(self.X_test, self.y_test)
+        self.assertTrue(isinstance(eval_dict, dict))
+        self.assertEqual(len(eval_dict.keys()), 13)
+
+
+if __name__ == "__main__":
+    unittest.main()

tinyconformal-0.1.0/tinyconformal/classifier/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2024-2026 Lucas Leão
+# tinyCP - A small toolbox for conformal prediction
+# Licensed under the MIT License
+
+
+from .marginal import BinaryMarginalConformalClassifier
+from .class_conditional import BinaryClassConditionalConformalClassifier