mlquantify 0.0.11.2__py3-none-any.whl → 0.1.0__py3-none-any.whl

mlquantify/evaluation/protocol/_Protocol.py

@@ -1,202 +0,0 @@
-from abc import ABC, abstractmethod
-import numpy as np
-import pandas as pd
-from typing import Union, List
-from sklearn.base import BaseEstimator
-from time import time
-from tqdm import tqdm
-
-from ...methods import get_method, METHODS, AGGREGATIVE, NON_AGGREGATIVE
-from ...utils import *
-from ..measures import get_measure, MEASURES
-from ...base import Quantifier, AggregativeQuantifier
-
-class Protocol(ABC):
-    """Base class for implementing different quantification protocols.
-
-    This abstract class provides a structure for creating protocols that involve
-    fitting quantification models to training data and generating predictions on test data.
-    It supports parallel processing, multiple iterations, and different output formats.
-
-    Args:
-        models (Union[List[Union[str, Quantifier]], str, Quantifier]): 
-            List of quantification models, a single model name, or 'all' for all models.
-        batch_size (Union[List[int], int]): 
-            Size of the batches to be processed, or a list of sizes.
-        learner (BaseEstimator, optional): 
-            Machine learning model to be used with the quantifiers. Required for model methods.
-        n_iterations (int, optional): 
-            Number of iterations for the protocol. Default is 1.
-        n_jobs (int, optional): 
-            Number of jobs to run in parallel. Default is 1.
-        random_state (int, optional): 
-            Seed for random number generation. Default is 32.
-        verbose (bool, optional): 
-            Whether to print progress messages. Default is False.
-        return_type (str, optional): 
-            Type of return value ('predictions' or 'table'). Default is 'predictions'.
-        measures (List[str], optional): 
-            List of error measures to calculate. Must be in MEASURES or None. Default is None.
-    """
-
-
-    def __init__(self,     
-                 models: Union[List[Union[str, Quantifier]], str, Quantifier], 
-                 batch_size: Union[List[int], int],
-                 learner: BaseEstimator = None, 
-                 n_iterations: int = 1,
-                 n_jobs: int = 1,
-                 random_state: int = 32,
-                 verbose: bool = False,
-                 return_type: str = "predictions",
-                 measures: List[str] = None):
-
-        assert not measures or all(m in MEASURES for m in measures), \
-            f"Invalid measure(s) provided. Valid options: {list(MEASURES.keys())} or None"
-        assert return_type in ["predictions", "table"], \
-            "Invalid return_type. Valid options: ['predictions', 'table']"
-
-        self.models = self._initialize_models(models, learner)
-        self.learner = learner
-        self.batch_size = batch_size
-        self.n_iterations = n_iterations
-        self.n_jobs = n_jobs
-        self.random_state = random_state
-        self.verbose = verbose
-        self.return_type = return_type
-        self.measures = measures
-
-    def _initialize_models(self, models, learner):
-        if isinstance(models, list):
-            if isinstance(models[0], Quantifier):
-                return models
-            assert learner is not None, "Learner is required for model methods."
-            return [get_method(model)(learner) for model in models]
-        if isinstance(models, Quantifier):
-            return [models]
-        
-        assert learner is not None, "Learner is required for model methods."
-        
-        if models == "all":
-            print(hasattr(list(AGGREGATIVE.values())[0], "learner"))  
-            models = [model(learner) if hasattr(model, "learner") else model() for model in METHODS.values()]
-            return models
-        if models == "aggregative":
-            return [model(learner) for model in AGGREGATIVE.values()]
-        if models == "non_aggregative":
-            return [model() for model in NON_AGGREGATIVE.values()]
-            
-        return [get_method(models)(learner)]
-    
-    
-    def sout(self, msg):
-        if self.verbose:
-            print('[APP]' + msg)    
-    
-        
-    def fit(self, X_train, y_train):
-        """Fit all methods into the training data.
-
-        Args:
-            X_train (array-like): Features of training.
-            y_train (array-like): Labels of training.
-        """
-        self.sout("Fitting models")
-
-        args = ((model, X_train, y_train, self.verbose) for model in self.models)
-        self.models = parallel(
-            self._delayed_fit, 
-            tqdm(args, desc="Fitting models", total=len(self.models)) if self.verbose else args, 
-            self.n_jobs)
-        
-        self.sout("Fit [Done]")
-        return self
-    
-    
-    def predict(self, X_test, y_test) -> np.any:
-        """Generate several samples with artificial prevalences, and sizes. 
-        And for each method, predicts with this sample, aggregating all toguether
-        with a pandas dataframe if request, or else just the predictions.
-
-        Args:
-            X_test (array-like): Features of test.
-            y_test (array-like): Labels of test.
-
-        Returns:
-            tuple: tuple containing the model, real_prev and pred_prev, or.
-            DataFrame: table of results, along with error measures if requested. 
-        """
-        
-        
-        predictions = self.predict_protocol(X_test, y_test) 
-
-
-        predictions_df = pd.DataFrame(predictions)
-        
-        if self.return_type == "table":
-            predictions_df.columns = ["QUANTIFIER", "REAL_PREVS", "PRED_PREVS", "BATCH_SIZE"]
-            
-            if self.measures:
-                
-                def smooth(values:np.ndarray) ->np.ndarray:
-                    smoothed_factor = 1/(2 * len(X_test))
-                    
-                    values = (values + smoothed_factor) / (smoothed_factor * len(values) + 1)
-
-                    return values
-                
-                
-                for metric in self.measures:
-                    predictions_df[metric] = predictions_df.apply(
-                        lambda row: get_measure(metric)(smooth(row["REAL_PREVS"]), smooth(row["PRED_PREVS"])),
-                        axis=1
-                    )
-            
-            return predictions_df
-        
-        predictions_array = predictions_df.to_numpy()
-        return (
-            predictions_array[:, 0],  # Model names
-            np.stack(predictions_array[:, 1]),  # Prev
-            np.stack(predictions_array[:, 2])   # Prev_pred
-        )
-        
-        
-    @abstractmethod
-    def predict_protocol(self) -> np.ndarray:
-        """ Abstract method that every protocol has to implement """
-        ...
-        
-    @abstractmethod
-    def _new_sample(self) -> tuple:
-        """ Abstract method of sample extraction for each protocol
-
-        Returns:
-            tuple: tuple containing the X_sample and the y_sample
-        """
-        ...
-        
-        
-    @abstractmethod
-    def _delayed_predict(self, args) -> tuple:
-        """abstract method for predicting in the extracted
-        samples, is delayed for running in parallel for 
-        eficciency purposes.
-        """
-        ...
-        
-
-    
-    def _delayed_fit(self, args):
-        model, X_train, y_train, verbose = args
-        
-        if verbose:
-            print(f"\tFitting {model.__class__.__name__}")
-            start = time()
-        
-        model = model.fit(X=X_train, y=y_train)
-        
-        if verbose:
-            end = time()
-            print(f"\t\\--Fit ended for {model.__class__.__name__} in {round(end - start, 3)} seconds")
-        return model

mlquantify/evaluation/protocol/__init__.py

@@ -1,2 +0,0 @@
-from .app import APP
-from .npp import NPP

mlquantify/evaluation/protocol/app.py

@@ -1,146 +0,0 @@
-import numpy as np
-import pandas as pd
-from typing import Union, List
-from sklearn.base import BaseEstimator
-import itertools
-from tqdm import tqdm
-
-from ...utils import generate_artificial_indexes, parallel
-from ...base import Quantifier
-from ._Protocol import Protocol
-
-class APP(Protocol):
-    """Artificial Prevalence Protocol. It splits a test into several
-    samples varying prevalence and sample size, with n iterations.
-    For a list of Quantifiers, it computes training and testing 
-    for each one and returns either a table of results with error measures
-    or just the predictions.
-    """
-    
-    def __init__(self,     
-                 models: Union[List[Union[str, Quantifier]], str, Quantifier], 
-                 batch_size: Union[List[int], int],
-                 learner: BaseEstimator = None, 
-                 n_prevs: int = 100,
-                 n_iterations: int = 1,
-                 n_jobs: int = 1,
-                 random_state: int = 32,
-                 verbose: bool = False,
-                 return_type: str = "predictions",
-                 measures: List[str] = None):
-        
-        super().__init__(models, batch_size, learner, n_iterations, n_jobs, random_state, verbose, return_type, measures)
-        self.n_prevs = n_prevs
-
-    def predict_protocol(self, X_test, y_test) -> tuple:
-        """Generates several samples with artificial prevalences and sizes.
-        For each model, predicts with this sample, aggregating all together
-        with a pandas dataframe if requested, or else just the predictions.
-
-        Args:
-            X_test (array-like): Features of the test set.
-            y_test (array-like): Labels of the test set.
-
-        Returns:
-            tuple: predictions containing the model name, real prev, pred prev, and batch size
-        """
-        
-        n_dim = len(np.unique(y_test))
-        prevs = self._generate_artificial_prevalences(n_dim, self.n_prevs, self.n_iterations)
-
-        args = self._generate_args(X_test, y_test, prevs)
-        batch_size = 1
-        
-        if isinstance(self.batch_size, list):
-            batch_size = len(self.batch_size)
-        
-        size = len(prevs) * len(self.models) * batch_size * self.n_iterations
-        
-        predictions = parallel(
-            self._delayed_predict,
-            tqdm(args, desc="Running APP", total=size) if self.verbose else args,
-            n_jobs=self.n_jobs
-        )
-        
-        return predictions
-
-
-    def _new_sample(self, X, y, prev: List[float], batch_size: int) -> tuple:
-        """Generates a new sample with a specified prevalence and size.
-
-        Args:
-            X (array-like): Features from which to take the new sample.
-            y (array-like): Labels from which to take the new sample.
-            prev (List[float]): The specified prevalences.
-            batch_size (int): Sample size.
-
-        Returns:
-            tuple: New sample's features and labels.
-        """
-        sample_index = generate_artificial_indexes(y, prev, batch_size, np.unique(y))
-        return np.take(X, sample_index, axis=0), np.take(y, sample_index, axis=0)
-
-
-
-    def _delayed_predict(self, args) -> tuple:
-        """Method predicts into the new sample, is delayed for running 
-        in parallel for eficciency purposes
-
-        Args:
-            args (Any): arguments to use 
-
-        Returns:
-            tuple: returns the (method name, real_prev, pred_prev and sample_size)
-        """
-        
-        X, y, model, prev, batch_size, verbose = args
-        
-        if verbose:
-            print(f'\t {model.__class__.__name__} with {str(batch_size)} instances and prev {str(prev)}')
-        
-        X_sample, _ = self._new_sample(X, y, prev, batch_size)
-        prev_pred = np.asarray(list(model.predict(X=X_sample).values()))
-        
-        if verbose:
-            print(f'\t \\--Ending {model.__class__.__name__} with {str(batch_size)} instances and prev {str(prev)} \n')
-        
-        return [model.__class__.__name__, prev, prev_pred, batch_size]
-    
-    
-
-
-    def _generate_artificial_prevalences(self, n_dim: int, n_prev: int, n_iter: int) -> np.ndarray:
-        """Generates n artificial prevalences with n dimensions.
-
-        Args:
-            n_dim (int): Number of dimensions for the artificial prevalence.
-            n_prev (int): Number of prevalence points to generate.
-            n_iter (int): Number of iterations.
-
-        Returns:
-            np.ndarray: Generated artificial prevalences.
-        """
-        s = np.linspace(0., 1., n_prev, endpoint=True)
-        prevs = np.array([p + (1 - sum(p),) for p in itertools.product(*(s,) * (n_dim - 1)) if sum(p) <= 1])
-        
-        return np.repeat(prevs, n_iter, axis=0) if n_iter > 1 else prevs
-
-
-
-    def _generate_args(self, X_test, y_test, prevs):
-        """Generates arguments for parallel processing based on the model, prevalence, and batch size.
-
-        Args:
-            X_test (array-like): Features of the test set.
-            y_test (array-like): Labels of the test set.
-            prevs (np.ndarray): Artificial prevalences generated.
-
-        Returns:
-            List[tuple]: List of arguments for parallel processing.
-        """
-        if isinstance(self.batch_size, list):
-            return [(X_test, y_test, model, prev, bs, self.verbose) 
-                    for prev in prevs for bs in self.batch_size for model in self.models]
-        else:
-            return [(X_test, y_test, model, prev, self.batch_size, self.verbose) 
-                    for prev in prevs for model in self.models]

mlquantify/evaluation/protocol/npp.py

@@ -1,34 +0,0 @@
-from typing import Union, List
-from sklearn.base import BaseEstimator
-
-from ...base import Quantifier
-from ._Protocol import Protocol
-
-
-class NPP(Protocol):
-    
-    
-    def __init__(self,     
-                 models: Union[List[Union[str, Quantifier]], str, Quantifier], 
-                 batch_size: Union[List[int], int],
-                 learner: BaseEstimator = None, 
-                 n_iterations: int = 1,
-                 n_jobs: int = 1,
-                 random_state: int = 32,
-                 verbose: bool = False,
-                 return_type: str = "predictions",
-                 measures: List[str] = None):
-        
-        super().__init__(models, batch_size, learner, n_iterations, n_jobs, random_state, verbose, return_type, measures)
-        
-    
-    def predict_protocol(self, X_test, y_test) -> tuple:
-        raise NotImplementedError
-        
-    
-    def _new_sample(self, X, y, prev: List[float], batch_size: int) -> tuple:
-        raise NotImplementedError
-        
-    
-    def _delayed_predict(self, args) -> tuple:
-        raise NotImplementedError

mlquantify/methods/aggregative/ThreholdOptm/_ThreholdOptimization.py

@@ -1,62 +0,0 @@
-from abc import abstractmethod
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ....base import AggregativeQuantifier
-from ....utils import adjust_threshold, get_scores
-
-class ThresholdOptimization(AggregativeQuantifier):
-    """Generic Class for methods that are based on adjustments
-    of the decision boundary of the underlying classifier in order
-    to make the ACC (base method for threshold methods) estimation
-    more numerically stable. Most of its strategies involve changing
-    the behavior of the denominator of the ACC equation.
-    """
-    # Class for optimizing classification thresholds
-
-    def __init__(self, learner: BaseEstimator):
-        self.learner = learner
-        self.threshold = None
-        self.cc_output = None
-        self.tpr = None
-        self.fpr = None
-    
-    @property
-    def multiclass_method(self) -> bool:
-        """ All threshold Methods are binary or non multiclass """
-        return False
-    
-    def _fit_method(self, X, y):
-
-        y_labels, probabilities = get_scores(X, y, self.learner, self.cv_folds, self.learner_fitted)
-        
-        # Adjust thresholds and compute true and false positive rates
-        thresholds, tprs, fprs = adjust_threshold(y_labels, probabilities[:, 1], self.classes)
-        
-        # Find the best threshold based on TPR and FPR
-        self.threshold, self.tpr, self.fpr = self.best_tprfpr(thresholds, tprs, fprs)
-        
-        return self
-    
-    def _predict_method(self, X) -> dict:
-              
-        probabilities = self.learner.predict_proba(X)[:, 1]
-        
-        # Compute the classification count output
-        self.cc_output = len(probabilities[probabilities >= self.threshold]) / len(probabilities)
-        
-        # Calculate prevalence, ensuring it's within [0, 1]
-        if self.tpr - self.fpr == 0:
-            prevalence = self.cc_output
-        else:
-            # Equation of all threshold methods to compute prevalence
-            prevalence = np.clip((self.cc_output - self.fpr) / (self.tpr - self.fpr), 0, 1)
-        
-        prevalences = [1- prevalence, prevalence]
-
-        return np.asarray(prevalences)
-    
-    @abstractmethod
-    def best_tprfpr(self, thresholds: np.ndarray, tpr: np.ndarray, fpr: np.ndarray) -> float:
-        """Abstract method for determining the best TPR and FPR to use in the equation"""
-        ...

mlquantify/methods/aggregative/ThreholdOptm/__init__.py

@@ -1,7 +0,0 @@
-from .acc import ACC
-from .max import MAX
-from .x import X_method
-from .t50 import T50
-from .ms import MS
-from .ms2 import MS2
-from .pacc import PACC

mlquantify/methods/aggregative/ThreholdOptm/acc.py

@@ -1,27 +0,0 @@
-
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class ACC(ThresholdOptimization):
-    """ Adjusted Classify and Count or Adjusted Count. Is a 
-    base method for the threhold methods.
-        As described on the Threshold base class, this method 
-    estimate the true positive and false positive rates from
-    the training data and utilize them to adjust the output 
-    of the CC method.
-    """
-    
-    def __init__(self, learner:BaseEstimator, threshold:float=0.5):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-        self.threshold = threshold
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        # Get the tpr and fpr where the threshold is equal to the base threshold, default is 0.5
-        
-        tpr = tprs[thresholds == self.threshold][0]
-        fpr = fprs[thresholds == self.threshold][0]
-        return (self.threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/max.py

@@ -1,23 +0,0 @@
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class MAX(ThresholdOptimization):
-    """ Threshold MAX. This method tries to use the
-    threshold where it maximizes the difference between
-    tpr and fpr to use in the denominator of the equation.
-    """
-    
-    def __init__(self, learner:BaseEstimator):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        max_index = np.argmax(np.abs(tprs - fprs))
-        
-        threshold = thresholds[max_index]
-        tpr= tprs[max_index]
-        fpr = fprs[max_index]
-        return (threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/ms.py

@@ -1,21 +0,0 @@
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class MS(ThresholdOptimization):
-    """ Median Sweep. This method uses an
-    ensemble of such threshold-based methods and 
-    takes the median prediction.
-    """
-    
-    def __init__(self, learner:BaseEstimator, threshold:float=0.5):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-        self.threshold = threshold
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        tpr = np.median(tprs)
-        fpr = np.median(fprs)
-        return (self.threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/ms2.py

@@ -1,25 +0,0 @@
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class MS2(ThresholdOptimization):
-    """ Median Sweep 2. It relies on the same
-    strategy of the Median Sweep, but compute 
-    the median only for cases in which 
-    tpr -fpr > 0.25
-    """
-    
-    def __init__(self, learner:BaseEstimator):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        indices = np.where(np.abs(tprs - fprs) > 0.25)[0]
-    
-        threshold = np.median(thresholds[indices])
-        tpr = np.median(tprs[indices])
-        fpr = np.median(fprs[indices])
-        
-        return (threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/pacc.py

@@ -1,41 +0,0 @@
-
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class PACC(ThresholdOptimization):
-    """ Probabilistic Adjusted Classify and Count. 
-    This method adapts the AC approach by using average
-    classconditional confidences from a probabilistic 
-    classifier instead of true positive and false positive rates.
-    """
-    
-    def __init__(self, learner:BaseEstimator, threshold:float=0.5):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-        self.threshold = threshold
-    
-    def _predict_method(self, X):
-        prevalences = {}
-        
-        probabilities = self.learner.predict_proba(X)[:, 1]
-        
-        mean_scores = np.mean(probabilities)
-        
-        if self.tpr - self.fpr == 0:
-            prevalence = mean_scores
-        else:
-            prevalence = np.clip(abs(mean_scores - self.fpr) / (self.tpr - self.fpr), 0, 1)
-        
-        prevalences[self.classes[1]] = prevalence
-        prevalences[self.classes[0]] = 1 - prevalence
-
-        return prevalences
-    
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        tpr = tprs[thresholds == self.threshold][0]
-        fpr = fprs[thresholds == self.threshold][0]
-        return (self.threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/t50.py

@@ -1,21 +0,0 @@
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class T50(ThresholdOptimization):
-    """ Threshold 50. This method tries to
-    use the threshold where tpr = 0.5.
-    """
-    
-    def __init__(self, learner:BaseEstimator):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        min_index = np.argmin(np.abs(tprs - 0.5))
-        threshold = thresholds[min_index]
-        tpr = tprs[min_index]
-        fpr = fprs[min_index]
-        return (threshold, tpr, fpr)

mlquantify/methods/aggregative/ThreholdOptm/x.py

@@ -1,23 +0,0 @@
-import numpy as np
-from sklearn.base import BaseEstimator
-
-from ._ThreholdOptimization import ThresholdOptimization
-
-class X_method(ThresholdOptimization):
-    """ Threshold X. This method tries to
-    use the threshold where fpr = 1 - tpr
-    """
-    
-    def __init__(self, learner:BaseEstimator):
-        assert isinstance(learner, BaseEstimator), "learner object is not an estimator"
-        super().__init__(learner)
-    
-    
-    def best_tprfpr(self, thresholds:np.ndarray, tprs: np.ndarray, fprs: np.ndarray) -> tuple:
-        min_index = np.argmin(abs(1 - (tprs + fprs)))
-        
-        threshold = thresholds[min_index]
-        tpr = tprs[min_index]
-        fpr = fprs[min_index]
-        
-        return (threshold, tpr, fpr)

mlquantify/methods/aggregative/__init__.py

@@ -1,9 +0,0 @@
-from .ThreholdOptm import *
-from .mixtureModels import *
-from .cc import CC
-from .pcc import PCC
-from .gac import GAC
-from .gpac import GPAC
-from .fm import FM
-from .emq import EMQ
-from .pwk import PWK