mlquantify 0.1.8__py3-none-any.whl → 0.1.9__py3-none-any.whl

mlquantify/model_selection/_protocol.py

@@ -0,0 +1,358 @@
+import numpy as np
+
+from mlquantify.base import BaseQuantifier, ProtocolMixin
+from mlquantify.utils._constraints import Interval, Options
+from mlquantify.utils._sampling import (
+    get_indexes_with_prevalence, 
+    simplex_grid_sampling,
+    simplex_uniform_kraemer,
+    simplex_uniform_sampling,
+)
+from mlquantify.utils._validation import validate_data
+from abc import ABC, abstractmethod
+from logging import warning
+import numpy as np
+
+    
+class BaseProtocol(ProtocolMixin, BaseQuantifier):
+    """Base class for evaluation protocols.
+    
+    Parameters
+    ----------
+    batch_size : int or list of int
+        The size of the batches to be used in the evaluation.
+    random_state : int, optional
+        The random seed for reproducibility.
+
+    Attributes
+    ----------
+    n_combinations : int
+
+    Raises
+    ------
+    ValueError
+        If the batch size is not a positive integer or list of positive integers.
+
+    Notes
+    -----
+    This class serves as a base class for different evaluation protocols, each with its own strategy for splitting the data into batches.
+    
+    Examples
+    --------
+    >>> class MyCustomProtocol(Protocol):
+    ...     def _iter_indices(self, X: np.ndarray, y: np.ndarray):
+    ...         for batch_size in self.batch_size:
+    ...             yield np.random.choice(X.shape[0], batch_size, replace=True)
+    ...
+    >>> protocol = MyCustomProtocol(batch_size=100, random_state=42)
+    >>> for idx in protocol.split(X, y):
+    ...     # Train and evaluate model
+    ...     pass
+
+    """
+    
+    _parameter_constraints = {
+        "batch_size": [Interval(left=1, right=None, discrete=True)],
+        "random_state": [Interval(left=0, right=None, discrete=True)]
+    }
+
+    def __init__(self, batch_size, random_state=None, **kwargs):
+        if isinstance(batch_size, int):
+            self.n_combinations = 1
+        else:
+            self.n_combinations = len(batch_size)
+
+        self.batch_size = [batch_size] if isinstance(batch_size, int) else batch_size
+        self.random_state = random_state
+
+        for name, value in kwargs.items():
+            setattr(self, name, value)
+            if isinstance(value, list):
+                self.n_combinations *= len(value)
+            elif isinstance(value, (int, float)):
+                self.n_combinations *= value
+            else:
+                raise ValueError(f"Invalid argument {name}={value}: must be int/float or list of int/float.")
+                 
+
+    def split(self, X: np.ndarray, y: np.ndarray):
+        """
+        Split the data into samples for evaluation.
+
+        Parameters
+        ----------
+        X : np.ndarray
+            The input features.
+        y : np.ndarray
+            The target labels.
+
+        Yields
+        ------
+        Generator[np.ndarray, np.ndarray]
+            A generator that yields the indices for each split.
+        """
+        X, y = validate_data(self, X, y)
+        for idx in self._iter_indices(X, y):
+            if len(idx) > len(X):
+                warning(f"Batch size {len(idx)} exceeds dataset size {len(X)}. Replacement sampling will be used.")
+            yield idx
+
+
+    @abstractmethod
+    def _iter_indices(self, X, y):
+        """Abstract method to be implemented by subclasses to yield indices for each batch."""
+        pass
+    
+    def get_n_combinations(self):
+        """
+        Get the number of combinations for the current protocol.
+        """
+        return self.n_combinations
+
+
+
+# ===========================================
+# Protocol Implementations
+# ===========================================
+
+
+class APP(BaseProtocol):
+    """
+    Artificial Prevalence Protocol (APP) for exhaustive prevalent batch evaluation.
+    
+    Generates batches with artificially imposed prevalences across all possible 
+    combinations within specified bounds. This allows comprehensive evaluation
+    over a range of prevalence scenarios.
+
+    Parameters
+    ----------
+    batch_size : int or list of int
+        Size(s) of the evaluation batches.
+    n_prevalences : int
+        Number of artificial prevalence levels to sample per class dimension.
+    repeats : int, optional (default=1)
+        Number of repetitions for each prevalence sampling.
+    random_state : int, optional
+        Random seed for reproducibility.
+    min_prev : float, optional (default=0.0)
+        Minimum possible prevalence for any class.
+    max_prev : float, optional (default=1.0)
+        Maximum possible prevalence for any class.
+
+    Notes
+    -----
+    For multiclass problems, this protocol may have high computational complexity
+    due to combinatorial explosion in prevalence combinations.
+
+    Examples
+    --------
+    >>> protocol = APP(batch_size=[100, 200], n_prevalences=5, repeats=3, random_state=42)
+    >>> for idx in protocol.split(X, y):
+    ...     # Train and evaluate
+    ...     pass
+    """
+    
+    _parameter_constraints = {
+        "n_prevalences": [Interval(left=1, right=None, discrete=True)],
+        "repeats": [Interval(left=1, right=None, discrete=True)],
+        "min_prev": [Interval(left=0.0, right=1.0)],
+        "max_prev": [Interval(left=0.0, right=1.0)]
+    }
+
+    def __init__(self, batch_size, n_prevalences, repeats=1, random_state=None, min_prev=0.0, max_prev=1.0):
+        super().__init__(batch_size=batch_size, 
+                            random_state=random_state,
+                            n_prevalences=n_prevalences, 
+                            repeats=repeats)
+        self.min_prev = min_prev
+        self.max_prev = max_prev
+
+    def _iter_indices(self, X: np.ndarray, y: np.ndarray):
+        
+        n_dim = len(np.unique(y))
+        
+        for batch_size in self.batch_size:
+            prevalences = simplex_grid_sampling(n_dim=n_dim,
+                                              n_prev=self.n_prevalences,
+                                              n_iter=self.repeats,
+                                              min_val=self.min_prev,
+                                              max_val=self.max_prev)
+            for prev in prevalences:
+                indexes = get_indexes_with_prevalence(y, prev, batch_size)
+                yield indexes
+                
+
+            
+
+class NPP(BaseProtocol):
+    """
+    Natural Prevalence Protocol (NPP) that samples data without imposing prevalence constraints.
+    
+    This protocol simply samples batches randomly with replacement, 
+    ignoring prevalence distributions.
+
+    Parameters
+    ----------
+    batch_size : int or list of int
+        Size(s) of the evaluation batches.
+    n_samples : int, optional (default=1)
+        Number of distinct batch samples per batch size.
+    repeats : int, optional (default=1)
+        Number of repetitions for each batch sample.
+    random_state : int, optional
+        Random seed for reproducibility.
+
+    Examples
+    --------
+    >>> protocol = NPP(batch_size=100, random_state=42)
+    >>> for idx in protocol.split(X, y):
+    ...     # Train and evaluate
+    ...     pass
+    """
+    
+    _parameter_constraints = {
+        "repeats": [Interval(left=1, right=None, discrete=True)]
+    }
+    
+    def __init__(self, batch_size, n_samples=1, repeats=1, random_state=None):
+        super().__init__(batch_size=batch_size, 
+                        random_state=random_state)
+        self.n_samples = n_samples
+        self.repeats = repeats
+
+    def _iter_indices(self, X: np.ndarray, y: np.ndarray):
+        
+        for _ in range(self.n_samples):
+            for batch_size in self.batch_size:
+                idx = np.random.choice(X.shape[0], batch_size, replace=True)
+                for _ in range(self.repeats):
+                    yield idx
+            
+
+class UPP(BaseProtocol):
+    """
+    Uniform Prevalence Protocol (UPP) for uniform sampling of artificial prevalences.
+    
+    Similar to APP, but uses uniform prevalence distribution generation
+    methods such as Kraemer or uniform simplex sampling to generate batches
+    with uniformly sampled class prevalences.
+
+    Parameters
+    ----------
+    batch_size : int or list of int
+        Batch size(s) for evaluation.
+    n_prevalences : int
+        Number of prevalence samples per class.
+    repeats : int
+        Number of evaluation repeats with different samples.
+    random_state : int, optional
+        Random seed for reproducibility.
+    min_prev : float, optional (default=0.0)
+        Minimum prevalence limit.
+    max_prev : float, optional (default=1.0)
+        Maximum prevalence limit.
+    algorithm : {'kraemer', 'uniform'}, optional (default='kraemer')
+        Sampling algorithm used to generate artificial prevalences.
+
+    Examples
+    --------
+    >>> protocol = UPP(batch_size=100, n_prevalences=5, repeats=3, random_state=42)
+    >>> for idx in protocol.split(X, y):
+    ...     # Train and evaluate
+    ...     pass
+    """
+    
+    _parameter_constraints = {
+        "n_prevalences": [Interval(left=1, right=None, discrete=True)],
+        "repeats": [Interval(left=1, right=None, discrete=True)],
+        "min_prev": [Interval(left=0.0, right=1.0)],
+        "max_prev": [Interval(left=0.0, right=1.0)],
+        "algorithm": [Options(['kraemer', 'uniform'])]
+    }
+
+    def __init__(self, 
+                 batch_size, 
+                 n_prevalences, 
+                 repeats=1, 
+                 random_state=None, 
+                 min_prev=0.0, 
+                 max_prev=1.0,
+                 algorithm='kraemer'):
+        super().__init__(batch_size=batch_size, 
+                            random_state=random_state,
+                            n_prevalences=n_prevalences, 
+                            repeats=repeats)
+        self.min_prev = min_prev
+        self.max_prev = max_prev
+        self.algorithm = algorithm
+
+    def _iter_indices(self, X: np.ndarray, y: np.ndarray):
+        
+        n_dim = len(np.unique(y))
+        
+        for batch_size in self.batch_size:
+            if self.algorithm == 'kraemer':
+                prevalences = simplex_uniform_kraemer(n_dim=n_dim,
+                                           n_prev=self.n_prevalences,
+                                           n_iter=self.repeats,
+                                           min_val=self.min_prev,
+                                           max_val=self.max_prev)
+            elif self.algorithm == 'uniform':
+                prevalences = simplex_uniform_sampling(n_dim=n_dim,
+                                              n_prev=self.n_prevalences,
+                                              n_iter=self.repeats,
+                                              min_val=self.min_prev,
+                                              max_val=self.max_prev)
+
+            for prev in prevalences:
+                indexes = get_indexes_with_prevalence(y, prev, batch_size)
+                yield indexes
+
+
+class PPP(BaseProtocol):
+    """
+    Personalized Prevalence Protocol (PPP) for targeted prevalence batch generation.
+    
+    Generates batches with user-specified prevalence distributions, allowing for
+    controlled evaluation on specific scenarios.
+
+    Parameters
+    ----------
+    batch_size : int or list of int
+        Batch sizes to generate.
+    prevalences : list of floats or array-like
+        Custom target prevalences per class to generate evaluation batches.
+    repeats : int, optional (default=1)
+        Number of evaluation repetitions with different batches.
+    random_state : int, optional
+        Random seed for reproducibility.
+
+    Examples
+    --------
+    >>> protocol = PPP(batch_size=100, prevalences=[0.1, 0.9], repeats=3, random_state=42)
+    >>> for idx in protocol.split(X, y):
+    ...     # Train and evaluate
+    ...     pass
+    """
+    
+    _parameter_constraints = {
+        "repeats": [Interval(left=1, right=None, discrete=True)],
+        "prevalences": ["array-like"]
+    }
+    
+    def __init__(self, batch_size, prevalences, repeats=1, random_state=None):
+        super().__init__(batch_size=batch_size, 
+                        random_state=random_state,
+                        prevalences=prevalences, 
+                        repeats=repeats)
+    
+    def _iter_indices(self, X: np.ndarray, y: np.ndarray):
+        
+        for batch_size in self.batch_size:    
+            for prev in self.prevalences:
+                if isinstance(prev, float):
+                    prev = [1-prev, prev]
+                
+                indexes = get_indexes_with_prevalence(y, prev, batch_size)
+                yield indexes
+        

mlquantify/model_selection/_search.py

@@ -0,0 +1,315 @@
+from mlquantify.base import BaseQuantifier, MetaquantifierMixin
+import itertools
+from joblib import Parallel, delayed
+from copy import deepcopy
+import numpy as np
+from sklearn.model_selection import train_test_split
+from mlquantify.metrics._slq import MAE
+from mlquantify.utils._constraints import (
+    Interval,
+    Options,
+    CallableConstraint
+)
+from mlquantify.utils._validation import validate_data
+from mlquantify.utils._decorators import _fit_context
+from mlquantify.utils.prevalence import get_prev_from_labels
+from mlquantify.model_selection import (
+    APP, NPP, UPP
+)
+
+class GridSearchQ(MetaquantifierMixin, BaseQuantifier):
+    """
+    Grid Search for Quantifiers with evaluation protocols.
+
+    This class automates the hyperparameter search over a grid of parameter
+    combinations for a given quantifier. It evaluates each combination using
+    a specified evaluation protocol (e.g., APP, NPP, UPP), over multiple splits
+    of the validation data, and selects the best-performing parameters based on
+    a chosen scoring metric such as Mean Absolute Error (MAE).
+
+    Parameters
+    ----------
+    quantifier : BaseQuantifier
+        Quantifier class (not instance). It must implement fit and predict.
+    param_grid : dict
+        Dictionary where keys are parameter names and values are lists of parameter
+        values to try.
+    protocol : {'app', 'npp', 'upp'}, default='app'
+        Evaluation protocol to use for splitting the validation data.
+    samples_sizes : int or list of int, default=100
+        Batch size(s) for evaluation splits.
+    n_repetitions : int, default=10
+        Number of random repetitions per evaluation.
+    scoring : callable, default=MAE
+        Scoring function to evaluate prevalence prediction quality.
+        Must accept (true_prevalences, predicted_prevalences) arrays.
+    refit : bool, default=True
+        If True, refits the quantifier on the whole data using best parameters.
+    val_split : float, default=0.4
+        Fraction of data reserved for validation during parameter search.
+    n_jobs : int or None, default=1
+        Number of parallel jobs for evaluation.
+    random_seed : int, default=42
+        Random seed for reproducibility.
+    verbose : bool, default=False
+        Enable verbose output during evaluation.
+
+    Attributes
+    ----------
+    best_score : float
+        Best score (lowest loss) found during grid search.
+    best_params : dict
+        Parameter combination corresponding to best_score.
+    best_model_ : BaseQuantifier
+        Refitted quantifier instance with best parameters after search.
+
+    Methods
+    -------
+    fit(X, y)
+        Runs grid search over param_grid, evaluates with the selected protocol,
+        and stores best found parameters and model.
+    predict(X)
+        Predicts prevalences using the best fitted model after search.
+    best_params()
+        Returns the best parameter dictionary after fitting.
+    best_model()
+        Returns the best refitted quantifier after fitting.
+    sout(msg)
+        Utility method to print messages if verbose is enabled.
+
+    Examples
+    --------
+    >>> from mlquantify.quantifiers import SomeQuantifier
+    >>> param_grid = {'alpha': [0.1, 1.0], 'beta': [10, 20]}
+    >>> grid_search = GridSearchQ(quantifier=SomeQuantifier,
+    ...                          param_grid=param_grid,
+    ...                          protocol='app',
+    ...                          samples_sizes=100,
+    ...                          n_repetitions=5,
+    ...                          scoring=MAE,
+    ...                          refit=True,
+    ...                          val_split=0.3,
+    ...                          n_jobs=2,
+    ...                          random_seed=123,
+    ...                          verbose=True)
+    >>> grid_search.fit(X_train, y_train)
+    >>> y_pred = grid_search.predict(X_test)
+    >>> best_params = grid_search.best_params()
+    >>> best_model = grid_search.best_model()
+    """
+    
+    _parameter_constraints = {
+        "quantifier": [BaseQuantifier],
+        "param_grid": [dict],
+        "protocol": [Options({'app', 'npp', 'upp'})],
+        "n_samples": [Interval(1, None)],
+        "n_repetitions": [Interval(1, None)],
+        "scoring": [CallableConstraint()],
+        "refit": [bool],
+        "val_split": [Interval(0.0, 1.0, inclusive_left=False, inclusive_right=False)],
+        "n_jobs": [Interval(1, None), None],
+        "random_seed": [Interval(0, None), None],
+        "timeout": [Interval(-1, None)],
+        "verbose": [bool]
+    }
+    
+    
+    def __init__(self,
+                 quantifier,
+                 param_grid,
+                 protocol="app",
+                 samples_sizes=100,
+                 n_repetitions=10,
+                 scoring=MAE,
+                 refit=True,
+                 val_split=0.4,
+                 n_jobs=1,
+                 random_seed=42,
+                 verbose=False):
+                     
+        self.quantifier = quantifier()
+        self.param_grid = param_grid
+        self.protocol = protocol.lower()
+        self.samples_sizes = samples_sizes
+        self.n_repetitions = n_repetitions
+        self.refit = refit
+        self.val_split = val_split
+        self.n_jobs = n_jobs
+        self.random_seed = random_seed
+        self.verbose = verbose
+        self.scoring = scoring
+        
+    
+    def sout(self, msg):
+        """Prints messages if verbose is True."""
+        if self.verbose:
+            print(f'[{self.__class__.__name__}]: {msg}')
+            
+    def __get_protocol(self):
+        
+        if self.protocol == "app":
+            return APP(batch_size=self.samples_sizes,
+                       n_prevalences=self.n_repetitions,
+                       repeats=self.n_repetitions,
+                       random_state=self.random_seed,
+                       min_prev=0.0,
+                       max_prev=1.0)
+        elif self.protocol == "npp":
+            return NPP(batch_size=self.samples_sizes,
+                       n_samples=self.n_repetitions,
+                       repeats=self.n_repetitions,
+                       random_state=self.random_seed)
+        elif self.protocol == "upp":
+            return UPP(batch_size=self.samples_sizes,
+                       n_prevalences=self.n_repetitions,
+                       repeats=self.n_repetitions,
+                       random_state=self.random_seed,
+                       min_prev=0.0,
+                       max_prev=1.0)
+        else:  
+            raise ValueError(f'Unknown protocol: {self.protocol}')
+        
+    @_fit_context(prefer_skip_nested_validation=True)
+    def fit(self, X, y):
+        """
+        Fit quantifiers over grid parameter combinations with evaluation protocol.
+
+        Splits data into training and validation by val_split, and evaluates
+        each parameter combination multiple times with protocol-generated batches.
+
+        Parameters
+        ----------
+        X : array-like
+            Feature matrix for training.
+        y : array-like
+            Target labels for training.
+
+        Returns
+        -------
+        self : object
+            Returns self for chaining.
+        """
+        X, y = validate_data(self, X, y)
+        
+        
+        X_train, X_val, y_train, y_val = train_test_split(X, y, test_size=self.val_split, random_state=self.random_seed)
+        param_combinations = list(itertools.product(*self.param_grid.values()))
+        params = list(self.param_grid.keys())
+
+        best_score, best_params = None, None
+        
+        def evaluate_combination(params):
+           
+            self.sout(f'Evaluating combination: {str(params)}')
+            
+            errors = []
+            
+            params = dict(zip(self.param_grid.keys(), params))
+
+            model = deepcopy(self.quantifier)
+            model.set_params(**params)
+            
+            protocol = self.__get_protocol()
+            
+            model.fit(X_train, y_train)
+            
+            for idx in protocol.split(X_val, y_val):
+                X_batch, y_batch = X_val[idx], y_val[idx]
+
+                y_real = get_prev_from_labels(y_batch)
+                y_pred = model.predict(X_batch)
+
+                
+                errors.append(self.scoring(y_real, y_pred))
+
+            avg_score = np.mean(errors)
+            
+            self.sout(f'\\--Finished evaluation: {str(params)} with score: {avg_score}')
+            
+            return avg_score
+                
+        
+        results = Parallel(n_jobs=self.n_jobs)(
+            delayed(evaluate_combination)(params) for params in param_combinations
+        )
+        
+            
+        for score, params in zip(results, param_combinations):
+            if score is not None and (best_score is None or score < best_score):
+                best_score, best_params = score, params
+            
+        
+        self.best_score = best_score
+        self.best_params = dict(zip(self.param_grid.keys(), best_params))
+        self.sout(f'Optimization complete. Best score: {self.best_score}, with parameters: {self.best_params}.')
+
+        if self.refit and self.best_params:
+            model = deepcopy(self.quantifier)
+            model.set_params(**self.best_params)
+            model.fit(X, y)
+            self.best_model_ = model
+
+        return self
+    
+    
+    
+    def predict(self, X):
+        """
+        Predict using the best found model.
+
+        Parameters
+        ----------
+        X : array-like
+            Data for prediction.
+
+        Returns
+        -------
+        predictions : array-like
+            Prevalence predictions.
+
+        Raises
+        ------
+        RuntimeError
+            If called before fitting.
+        """
+        if not hasattr(self, 'best_model_'):
+            raise RuntimeError("The model has not been fitted yet.")
+        return self.best_model_.predict(X)
+    
+    
+    
+    def best_params(self):
+        """Return the best parameters found during fitting.
+
+        Returns
+        -------
+        dict
+            The best parameters.
+
+        Raises
+        ------
+        ValueError
+            If called before fitting.
+        """
+        if hasattr(self, 'best_params'):
+            return self.best_params
+        raise ValueError('best_params called before fit.')
+        
+        
+        
+    def best_model(self):
+        """Return the best model after fitting.
+
+        Returns
+        -------
+        Quantifier
+            The best fitted model.
+
+        Raises
+        ------
+        ValueError
+            If called before fitting.
+        """
+        if hasattr(self, 'best_model_'):
+            return self.best_model_
+        raise ValueError('best_model called before fit.')

mlquantify/model_selection/_split.py

@@ -0,0 +1 @@
+# TODO