tinyconformal 0.1.0__py3-none-any.whl

tinyconformal/classifier/__init__.py

@@ -0,0 +1,7 @@
+# Copyright (c) 2024-2026 Lucas Leão
+# tinyCP - A small toolbox for conformal prediction
+# Licensed under the MIT License
+
+
+from .marginal import BinaryMarginalConformalClassifier
+from .class_conditional import BinaryClassConditionalConformalClassifier

tinyconformal/classifier/base.py

@@ -0,0 +1,433 @@
+# Copyright (c) 2024-2026 Lucas Leão
+# tinyCP - A small toolbox for conformal prediction
+# Licensed under the MIT License
+
+
+from venn_abers import VennAbers
+from sklearn.utils.validation import check_is_fitted
+from sklearn.base import BaseEstimator
+import warnings
+import numpy as np
+import sklearn.metrics
+from abc import ABC, abstractmethod
+
+warnings.filterwarnings("ignore", category=RuntimeWarning, module="venn_abers")
+
+
+class BaseConformalClassifier(ABC):
+    """
+    BaseConformalClassifier
+
+    A base class for conformal prediction using a model as the learner
+    and Venn-Abers calibration for confidence estimation.
+    This approach provides valid predictions with a specified significance level (alpha).
+
+    Conformal classifiers aim to quantify uncertainty in predictions.
+    """
+
+    def __init__(
+        self,
+        learner: BaseEstimator,
+        alpha: float = 0.05,
+    ):
+        """
+
+        Initializes the classifier with a specified learner and a Venn-Abers calibration layer.
+
+        Parameters
+        ----------
+        learner : BaseEstimator
+            The base learner to be used in the classifier.
+        alpha : float, default=0.05
+            The significance level applied in the classifier.
+
+        Attributes
+        ----------
+        learner : BaseEstimator
+            The base learner employed in the classifier.
+        calibration_layer : VennAbers
+            The calibration layer utilized in the classifier.
+        decision_function_ : callable or None
+            The decision function of the learner.
+        hinge : array-like of shape (n_samples,), default=None
+            The non-conformity scores of the calibration samples.
+        alpha : float, default=0.05
+            The significance level applied in the classifier.
+        n : int or None
+            The number of calibration samples.
+        """
+
+        self.learner = learner
+        self.alpha = alpha
+        self.calibration_layer = VennAbers()
+        self.classes = getattr(self.learner, "classes_", [0, 1])
+        self.decision_function_ = None
+        self.is_unlabeled = False
+        check_is_fitted(learner)
+
+        if learner.n_classes_ > 2:
+            raise ValueError("This classifier supports only binary classification.")
+
+        self.hinge = None
+        self.n = None
+
+    @abstractmethod
+    def fit(self, y):
+        """
+        Fits the classifier to the training data.
+        """
+        pass
+
+    @abstractmethod
+    def predict_set(self, X, alpha=None):
+        """
+        Generate a prediction set for the given input.
+        This method must be implemented by subclasses.
+        """
+        pass
+
+    @abstractmethod
+    def _compute_qhat(self, ncscore, q_level):
+        """
+        Compute the q-hat value based on the nonconformity scores and the quantile level.
+        """
+        pass
+
+    @abstractmethod
+    def _compute_set(self, ncscore, qhat):
+        """
+        Compute a set based on the given ncscore and qhat.
+        """
+        pass
+
+    @abstractmethod
+    def _compute_q_level(self, n, alpha):
+        """
+        Compute the quantile level based on the number of samples and significance level.
+        """
+        pass
+
+    def _compute_prediction(self, prediction_set):
+        """
+        Compute the prediction based on the given prediction set.
+
+        This method evaluates each row in the prediction set and returns 1 if all elements
+        in the row match the pattern [0, 1], otherwise returns 0.
+        """
+        return np.where(np.all(prediction_set == [0, 1], axis=1), 1, 0)
+
+    def _bookmaker_informedness(self, y, y_pred):
+        """
+        Calculate the bookmaker informedness score for the given true and predicted labels.
+        """
+        return sklearn.metrics.balanced_accuracy_score(y, y_pred, adjusted=True)
+
+    def _select_scoring_function(self, scoring_func):
+        """
+        Select the scoring function based on the provided string.
+        """
+
+        if scoring_func == "bm":
+            func = self._bookmaker_informedness
+        elif scoring_func == "mcc":
+            func = sklearn.metrics.matthews_corrcoef
+        else:
+            raise ValueError("Invalid metric function. Please use 'bm' or 'mcc'.")
+        return func
+
+    def _get_alpha(self, alpha):
+        """Helper to retrieve the alpha value."""
+        return alpha or self.alpha
+
+    def generate_non_conformity_score(self, y_prob):
+        """
+        Generates the non-conformity score based on the hinge loss.
+
+        This function calculates the non-conformity score for conformal prediction
+        using the hinge loss approach.
+        """
+        return 1 - y_prob
+
+    def generate_conformal_quantile(self, alpha=None):
+        """
+        Generate the conformal quantile for conformal prediction.
+
+        This method calculates the conformal quantile based on the nonconformity scores
+        of the calibration samples. The quantile serves as a threshold to determine
+        the prediction sets in conformal prediction.
+
+        Parameters:
+        -----------
+        alpha : float, optional
+            The significance level for conformal prediction. If None, the default
+            value of self.alpha is used.
+
+        Returns:
+        --------
+        float
+            The computed conformal quantile.
+
+        Notes:
+        ------
+        - The quantile is computed as ceil((n + 1) * (1 - alpha)) / n, where n is the
+          number of calibration samples.
+        - This method relies on the self.ncscore attribute, which should contain the
+          nonconformity scores of the calibration samples.
+        """
+
+        alpha = self._get_alpha(alpha)
+
+        q_level = self._compute_q_level(self.n, alpha)
+
+        return self._compute_qhat(self.hinge, q_level)
+
+    def predict_proba(self, X):
+        """
+        Returns class probabilities. Uses Venn-Abers if fit() was used,
+        or raw learner probabilities if unlabeled_fit() was used.
+
+        Parameters:
+        X: array-like of shape (n_samples, n_features)
+            The input samples.
+
+        Returns:
+        p_prime: array-like of shape (n_samples, n_classes)
+            The calibrated class probabilities.
+        """
+        y_score = self.learner.predict_proba(X)
+
+        if getattr(self, "is_unlabeled", True):
+            return y_score
+
+        p_prime, _ = self.calibration_layer.predict_proba(y_score)
+        return p_prime
+
+    def calibrate(self, X, y, max_alpha=0.2, func="mcc"):
+        """
+        Calibrates the alpha value to optimize the specified metric.
+
+        This method evaluates a range of alpha values (from 0.01 to `max_alpha`)
+        to determine the optimal significance level based on the provided scoring
+        function. The alpha value that maximizes the scoring function is selected.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input samples used for calibration.
+        y : array-like of shape (n_samples,)
+            True labels corresponding to the input samples.
+        max_alpha : float, optional, default=0.2
+            The maximum alpha value to consider during calibration. The range of
+            alpha values tested will be from 0.01 to `max_alpha`, inclusive.
+        func : str, optional, default="mcc"
+            The name of the scoring function to use for optimization. Supported
+            functions should be implemented in the `_select_scoring_function` method.
+
+        Raises
+        ------
+            If an invalid scoring function name is provided in the `func` parameter.
+
+        Returns
+            The optimal alpha value that maximizes the scoring function.
+        """
+
+        if getattr(self, "is_unlabeled", True):
+            raise ValueError(
+                "Calibration is not applicable for unlabeled data. Please use labeled data for calibration."
+            )
+
+        scoring_func = self._select_scoring_function(func)
+
+        alphas = {k: None for k in np.round(np.arange(0.01, max_alpha + 0.01, 0.01), 2)}
+
+        for alpha in alphas:
+            y_pred = self.predict(X, alpha)
+            alphas[alpha] = scoring_func(y, y_pred)
+
+        self.alpha = max(alphas, key=alphas.get)
+
+        return self.alpha
+
+    def predict(self, X, alpha=None):
+        """
+        Predicts the classes for the input samples.
+
+        Parameters:
+        -----------
+        X: np.ndarray of shape (n_samples, n_features)
+            Input samples.
+        alpha: float, optional
+            Significance level. If None, defaults to the classifier's alpha value.
+
+        Returns:
+        --------
+        np.ndarray of shape (n_samples,)
+            Predicted class labels, where 1 indicates the model's certainty.
+        """
+
+        alpha = self._get_alpha(alpha)
+
+        prediction_set = self.predict_set(X, alpha)
+
+        return self._compute_prediction(prediction_set)
+
+    def _expected_calibration_error(self, y, y_prob, M=5):
+        """
+        Generate the expected calibration error (ECE) of the classifier.
+
+        Parameters:
+        y: array-like of shape (n_samples,)
+            The true labels.
+        y_prob: array-like of shape (n_samples, n_classes)
+            The predicted probabilities.
+        M: int, default=5
+            The number of bins for the uniform binning approach.
+
+        Returns:
+        ece: float
+            The expected calibration error.
+
+        The function works as follows:
+        - It first creates M bins with uniform width over the interval [0, 1].
+        - For each sample, it computes the maximum predicted probability and makes a prediction.
+        - It then checks whether each prediction is correct or not.
+        - For each bin, it calculates the empirical probability of a sample falling into the bin.
+        - If the empirical probability is greater than 0, it computes the accuracy and average confidence of the bin.
+        - It then calculates the absolute difference between the accuracy and the average confidence, multiplies it by the empirical probability, and adds it to the total ECE.
+        """
+
+        # uniform binning approach with M number of bins
+        bin_boundaries = np.linspace(0, 1, M + 1)
+        bin_lowers = bin_boundaries[:-1]
+        bin_uppers = bin_boundaries[1:]
+
+        # get max probability per sample i
+        confidences = np.max(y_prob, axis=1)
+        # get predictions from confidences (positional in this case)
+        predicted_label = np.argmax(y_prob, axis=1)
+
+        # get a boolean list of correct/false predictions
+        predictions = predicted_label == y
+
+        ece = 0.0
+        for bin_lower, bin_upper in zip(bin_lowers, bin_uppers):
+            # determine if sample is in bin m (between bin lower & upper)
+            in_bin = np.logical_and(
+                confidences > bin_lower.item(), confidences <= bin_upper.item()
+            )
+            # can calculate the empirical probability of a sample falling into bin m: (|Bm|/n)
+            prob_in_bin = np.mean(in_bin)
+
+            if prob_in_bin > 0:
+                # get the accuracy of bin m: acc(Bm)
+                avg_pred = np.mean(predictions[in_bin])
+                # get the average confidence of bin m: conf(Bm)
+                avg_confidence_in_bin = np.mean(confidences[in_bin])
+                # calculate |acc(Bm) - conf(Bm)| * (|Bm|/n) for bin m and add to the total ECE
+                ece += np.abs(avg_pred - avg_confidence_in_bin) * prob_in_bin
+        return ece
+
+    def _false_positive_rate(self, y, y_pred):
+        """
+        Computes the false positive rate (FPR).
+        """
+        tn, fp, _, _ = sklearn.metrics.confusion_matrix(y, y_pred).ravel()
+        return fp / (fp + tn)
+
+    def _coverage_rate(self, X, y, alpha=None):
+        """
+        Compute the coverage rate from conformal prediction.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input features.
+        y : array-like of shape (n_samples,)
+            True labels.
+        alpha : float, optional
+            Significance level (1 - desired coverage). If None, the default value of self.alpha is used.
+
+        Returns
+        -------
+        float
+            The average coverage rate, which represents the proportion of true labels covered by the prediction sets.
+        """
+
+        alpha = self._get_alpha(alpha)
+        predict_sets = self.predict_set(X, alpha)
+        coverages = predict_sets[np.arange(len(y)), y]
+
+        return np.mean(coverages)
+
+    def evaluate(self, X, y, alpha=None):
+        """
+        Evaluate the classifier on the given dataset.
+
+        Parameters
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Input samples.
+        y : array-like of shape (n_samples,)
+            True labels for the input samples.
+        alpha : float, optional
+            Significance level for prediction sets. If None, the classifier's default alpha is used.
+
+        Returns
+        -------
+        results : dict
+            A dictionary containing the following evaluation metrics:
+            - "total": Total number of samples.
+            - "alpha": Significance level used.
+            - "coverage_rate": Coverage rate of the prediction sets.
+            - "one_c": Proportion of prediction sets containing exactly one element.
+            - "avg_c": Average size of the prediction sets.
+            - "empty": Proportion of empty prediction sets.
+            - "error": Classification error rate.
+            - "log_loss": Log loss of the predictions.
+            - "ece": Expected calibration error.
+            - "bm": Bookmaker informedness score.
+            - "mcc": Matthews correlation coefficient.
+            - "f1": F1 score.
+            - "fpr": False positive rate.
+        """
+
+        alpha = self._get_alpha(alpha)
+
+        # Helper function for rounding
+        def rounded(value):
+            return np.round(value, 3)
+
+        y_prob = self.predict_proba(X)
+        y_pred = self.predict(X, alpha)
+        predict_set = self.predict_set(X, alpha)
+        total = X.shape[0] if hasattr(X, "shape") else len(X)
+        coverage_rate = rounded(self._coverage_rate(X, y, alpha))
+        one_c = rounded(np.mean([np.sum(p) == 1 for p in predict_set]))
+        avg_c = rounded(np.mean([np.sum(p) for p in predict_set]))
+        empty = rounded(np.mean([np.sum(p) == 0 for p in predict_set]))
+        error = rounded(1 - np.mean(predict_set[np.arange(len(y)), y]))
+        log_loss = rounded(sklearn.metrics.log_loss(y, y_prob[:, 1]))
+        ece = rounded(self._expected_calibration_error(y, y_prob))
+        fpr = rounded(self._false_positive_rate(y, y_pred))
+        bookmaker_informedness = rounded(self._bookmaker_informedness(y, y_pred))
+        matthews_corr = rounded(sklearn.metrics.matthews_corrcoef(y, y_pred))
+        f1 = rounded(sklearn.metrics.f1_score(y, self.predict(X, alpha)))
+
+        # Results aggregation
+        results = {
+            "total": total,
+            "alpha": alpha,
+            "coverage_rate": coverage_rate,
+            "one_c": one_c,
+            "avg_c": avg_c,
+            "empty": empty,
+            "error": error,
+            "log_loss": log_loss,
+            "ece": ece,
+            "bm": bookmaker_informedness,
+            "mcc": matthews_corr,
+            "f1": f1,
+            "fpr": fpr,
+        }
+
+        return results

tinyconformal/classifier/class_conditional.py

@@ -0,0 +1,222 @@
+# Copyright (c) 2024-2026 Lucas Leão
+# tinyCP - A small toolbox for conformal prediction
+# Licensed under the MIT License
+
+
+from sklearn.base import ClassifierMixin, BaseEstimator
+import numpy as np
+import warnings
+from .base import BaseConformalClassifier
+
+warnings.filterwarnings("ignore", category=RuntimeWarning, module="venn_abers")
+
+
+class BinaryClassConditionalConformalClassifier(
+    ClassifierMixin, BaseEstimator, BaseConformalClassifier
+):
+    """
+    A modrian class conditional conformal classifier methodology utilizing a classifier as the underlying learner.
+    This class is inspired by the WrapperClassifier classes from the Crepes library.
+    """
+
+    def __init__(
+        self,
+        learner: BaseEstimator,
+        alpha: float = 0.05,
+    ):
+        """
+        Constructs the classifier with a specified learner and a Venn-Abers calibration layer.
+
+        Parameters:
+        ----------
+        learner : BaseEstimator
+            The base learner to be used in the classifier.
+        alpha : float, default=0.05
+            The significance level applied in the classifier.
+
+        Attributes:
+        ----------
+        learner : BaseEstimator
+            The base learner employed in the classifier.
+        calibration_layer : VennAbers
+            The calibration layer utilized in the classifier.
+        classes : array-like of shape (n_classes,), default=None
+            The unique class labels identified during training.
+        hinge : list of array-like, default=None
+            Nonconformity scores for each class based on the predicted probabilities.
+        n : array-like of shape (n_classes,), default=None
+            The number of calibration points for each class.
+        alpha : float, default=0.05
+            The significance level applied in the classifier.
+        """
+
+        super().__init__(learner, alpha)
+
+    def unlabeled_fit(self, X=None):
+        """
+        Fits the class-conditional conformal layer using unlabeled data (X) based on
+        pseudo-labels derived from the model's predictions (Flechsig & Pilz, 2025).
+
+        Parameters:
+        ----------
+        X : array-like of shape (n_samples, n_features)
+            Unlabeled calibration features.
+
+        Returns:
+        -------
+        self : object
+            The fitted classifier.
+        """
+        if X is None:
+            raise ValueError("Unlabeled calibration data (X) must be provided.")
+
+        self.is_unlabeled = True
+        y_prob = self.learner.predict_proba(X)
+        idx_max = np.argmax(y_prob, axis=1)
+        ncscore = np.min(self.generate_non_conformity_score(y_prob), axis=1)
+        self.hinge = [ncscore[idx_max == c] for c in self.classes]
+        self.n = [np.sum(idx_max == c) for c in self.classes]
+
+        return self
+
+    def fit(self, X=None, y=None, oob=False):
+        """
+        Fits the classifier to the training data. Calculates the conformity score for each training instance.
+
+        Parameters:
+        ----------
+        X : array-like of shape (n_samples, n_features), optional
+            The training data. Required if OOB predictions are not used.
+        y : array-like of shape (n_samples,)
+            The true labels. Required in all cases.
+        oob : bool, default=False
+            Whether to use Out-of-Bag (OOB) predictions if available.
+
+        Returns:
+        -------
+        self : object
+            The fitted classifier.
+
+        Raises:
+        ------
+        ValueError:
+            If OOB is enabled but not supported by the learner,
+            or if `X` and `y` are not provided when `oob=False`.
+        """
+        if y is None:
+            raise ValueError("The true labels (y) must be provided.")
+
+        if oob:
+            if (
+                not hasattr(self.learner, "oob_decision_function_")
+                or self.learner.oob_decision_function_ is None
+            ):
+                raise ValueError(
+                    "OOB predictions are not available for the provided learner."
+                )
+            if X is not None:
+                raise ValueError(
+                    "Training data (X) should not be provided when OOB is used. Ensure that 'y' is the same as the labels used during training."
+                )
+
+            # Use OOB predictions
+            self.decision_function_ = self.learner.oob_decision_function_
+        else:
+
+            if X is None:
+                raise ValueError(
+                    "Training data (X) must be provided if OOB is not used."
+                )
+
+            # Use predict_proba for training data
+            self.decision_function_ = self.learner.predict_proba(X)
+
+        self.calibration_layer.fit(self.decision_function_, y)
+
+        y_prob, _ = self.calibration_layer.predict_proba(self.decision_function_)
+
+        y_prob = y_prob[np.arange(len(y)), y]
+        hinge = self.generate_non_conformity_score(y_prob)
+        self.hinge = [hinge[y == c] for c in self.classes]
+        self.n = [np.sum(y == c) for c in self.classes]
+
+        return self
+
+    def _compute_q_level(self, n, alpha):
+        """
+        Compute the quantile level for each class based on the number of samples and significance level.
+        """
+        alpha = self._get_alpha(alpha)
+        q_level = np.zeros(len(self.classes))
+        for c in self.classes:
+            q_level[c] = np.ceil((n[c] + 1) * (1 - alpha)) / n[c]
+        return q_level
+
+    def _compute_qhat(self, ncscore, q_level):
+        """
+        Compute the q-hat value based on the nonconformity scores and the quantile level.
+        """
+        qhat = np.zeros(len(self.classes))
+        for c in self.classes:
+            qhat[c] = np.quantile(ncscore[c], q_level[c], method="higher")
+        return qhat
+
+    def _compute_set(self, ncscore, qhat):
+        """
+        Compute a predict set based on the given ncscore and qhat.
+        """
+        prediction_set = np.zeros((len(ncscore), len(self.classes)))
+        for c in self.classes:
+            prediction_set[:, c] = (ncscore <= qhat[c])[:, c]
+        return prediction_set
+
+    def predict_set(self, X, alpha=None):
+        """
+        Predicts the possible set of classes for the instances in X based on the predefined significance level.
+
+        Parameters:
+        X: array-like of shape (n_samples, n_features)
+            The input samples.
+        alpha: float, default=None
+            The significance level. If None, the value of self.alpha is used.
+
+        Returns:
+        prediction_set: array-like of shape (n_samples, n_classes)
+            The predicted set of classes. A class is included in the set if its non-conformity score is less
+            than or equal to the quantile of the hinge loss distribution at the (n+1)*(1-alpha)/n level.
+        """
+
+        alpha = self._get_alpha(alpha)
+
+        y_prob = self.predict_proba(X)
+        ncscore = self.generate_non_conformity_score(y_prob)
+        qhat = self.generate_conformal_quantile(alpha)
+
+        return self._compute_set(ncscore, qhat)
+
+    def predict_p(self, X):
+        """
+        Calculate the p-values for each instance in the input data X using a non-conformity score.
+
+        Parameters:
+        -----------
+        X : array-like of shape (n_samples, n_features)
+            The input data for which the p-values need to be predicted.
+
+        Returns:
+        --------
+        p_values : array-like of shape (n_samples, n_classes)
+            The p-values for each instance in X for each class.
+
+        """
+        y_prob = self.predict_proba(X)
+        ncscore = self.generate_non_conformity_score(y_prob)
+        p_values = np.zeros_like(ncscore)
+
+        for i in range(ncscore.shape[0]):
+            for j in range(ncscore.shape[1]):
+                numerator = np.sum(self.hinge[j] >= ncscore[i][j]) + 1
+                denumerator = self.n[j] + 1
+                p_values[i, j] = numerator / denumerator
+
+        return p_values