hypershap 0.0.1__py3-none-any.whl

hypershap/__init__.py

@@ -0,0 +1,18 @@
+"""HyperSHAP: Shapley values and interactions for explaining hyperparameter optimization.
+
+HyperSHAP is a package for explaining hyperparameter optimization, i.e., the effect of specific hyperparameter values,
+the tunability of machine learning algorithms, or biases in the optimizer. The package is based on the well established
+Shapley value and its generalization to interactions.
+"""
+
+from __future__ import annotations
+
+from .hypershap import HyperSHAP
+from .task import ExplanationTask
+from .utils import ConfigSpaceSearcher
+
+__all__ = [
+    "ConfigSpaceSearcher",
+    "ExplanationTask",
+    "HyperSHAP",
+]

hypershap/games/__init__.py

@@ -0,0 +1,18 @@
+"""The games module contains all the game-theoretic explanation games of HyperSHAP."""
+
+from __future__ import annotations
+
+from .ablation import AblationGame
+from .abstract import AbstractHPIGame
+from .optimizerbias import OptimizerBiasGame
+from .tunability import MistunabilityGame, SearchBasedGame, SensitivityGame, TunabilityGame
+
+__all__ = [
+    "AblationGame",
+    "AbstractHPIGame",
+    "MistunabilityGame",
+    "OptimizerBiasGame",
+    "SearchBasedGame",
+    "SensitivityGame",
+    "TunabilityGame",
+]

hypershap/games/ablation.py

@@ -0,0 +1,101 @@
+"""Module for ablation-based Hyperparameter Importance analysis.
+
+This module provides the `AblationGame` class for analyzing hyperparameter
+interactions through ablation-based experiments.  It leverages a surrogate model
+to estimate the impact of individual hyperparameters by systematically combining
+baseline and optimized configuration values.  The `AblationGame` allows for
+understanding which hyperparameters contribute most to performance gains and
+identifying potential areas for further tuning.
+
+The module is designed to work in conjunction with the `hypershap.games.abstract`
+module and relies on the `hypershap.task` module for defining the
+hyperparameter search space and surrogate model.
+
+Key components include:
+
+*   **`AblationGame`**: A game class that evaluates the performance of
+    configurations generated by blending baseline and optimized hyperparameter
+    values, providing insights into individual hyperparameter contributions.
+
+This module facilitates a deeper understanding of hyperparameter dependencies
+and guides optimization efforts by revealing the impact of each hyperparameter
+in a systematic and interpretable manner.
+"""
+
+from __future__ import annotations
+
+import numpy as np
+
+from hypershap.games.abstract import AbstractHPIGame
+from hypershap.task import AblationExplanationTask
+
+
+class AblationGame(AbstractHPIGame):
+    """The ablation game generates local explanations for hyperparameter configurations.
+
+    It does so by evaluating all potential ablation paths, switching from a baseline configuration
+    to an optimized configuration value by value. It leverages a surrogate model to estimate
+    performance based on blended configurations.
+    """
+
+    def __init__(
+        self,
+        explanation_task: AblationExplanationTask,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize an AblationGame.
+
+        Args:
+        explanation_task (AblationExplanationTask): An instance of `AblationExplanationTask`
+            providing access to the baseline configuration, configuration of interest,
+            and the surrogate model for evaluation.
+        n_workers (int | None): The number of worker threads to use for parallel
+            evaluation of coalitions. Defaults to None, which disables parallelization.
+            Using more workers can significantly speed up the computation of Shapley values.
+            The maximum number of workers is capped by the number of coalitions.
+        verbose (bool | None): A boolean indicating whether to print verbose messages
+            during computation. Defaults to None. When set to True, the method prints
+            debugging information and progress updates.
+
+        """
+        super().__init__(explanation_task, n_workers=n_workers, verbose=verbose)
+
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate a single coalition (combination of baseline and optimized hyperparameters).
+
+        This method blends the baseline and optimized configurations based on the provided coalition,
+        then uses the surrogate model to estimate the performance of the blended configuration.
+
+        Args:
+            coalition (np.ndarray): A Boolean array indicating which hyperparameters should be taken from the
+                baseline configuration (False) and which from the configuration of interest (True).
+
+        Returns:
+            float: The performance score of the blended configuration as estimated by the surrogate model.
+
+        """
+        baseline_cfg = self._get_explanation_task().baseline_config.get_array()
+        cfg_of_interest = self._get_explanation_task().config_of_interest.get_array()
+        blend = np.where(coalition == 0, baseline_cfg, cfg_of_interest)
+        res = self._get_explanation_task().surrogate_model.evaluate(blend)
+
+        # validate that we do not get a list of floats by accident
+        if isinstance(res, list):
+            raise TypeError
+
+        return res
+
+    def _get_explanation_task(self) -> AblationExplanationTask:
+        """Retrieve the explanation task associated with this ablation game.
+
+        This method simply returns the `explanation_task` attribute, which provides
+        access to the baseline configuration, configuration of interest, and surrogate model.
+
+        Returns:
+            AblationExplanationTask: The explanation task associated with this ablation game.
+
+        """
+        if isinstance(self.explanation_task, AblationExplanationTask):
+            return self.explanation_task
+        raise ValueError

hypershap/games/abstract.py

@@ -0,0 +1,161 @@
+"""The abstract module defines the abstract base class `AbstractHPIGame` for analyzing HPI.
+
+The `AbstractHPIGame` class provides a foundational structure for constructing
+and analyzing games that represent interactions between hyperparameters,
+enabling a deeper understanding of their impact on optimization performance.
+It handles initialization, normalization, and provides a framework for
+parallel evaluation of coalitions. Subclasses must implement the
+`evaluate_single_coalition` method to define the game-specific evaluation logic.
+"""
+
+from __future__ import annotations
+
+from abc import abstractmethod
+from concurrent.futures import ThreadPoolExecutor
+from typing import TYPE_CHECKING
+
+import numpy as np
+from shapiq import Game
+
+if TYPE_CHECKING:
+    from collections.abc import Iterable
+
+    from hypershap.task import ExplanationTask
+
+
+class AbstractHPIGame(Game):
+    """Abstract base class for Hyperparameter Importance Games (HPIGames).
+
+    Represents a game-theoretic framework for analyzing the importance of
+    hyperparameters for HPO. It leverages the `shapiq` library to compute
+    Shapley values and analyze coalition behavior.
+
+    Args:
+        explanation_task: The `ExplanationTask` containing information about
+            the configuration space and surrogate model.
+        n_workers: The number of worker threads to use for parallel evaluation
+            of coalitions. Defaults to 1.  Using more workers can significantly
+            speed up the computation of Shapley values.
+        verbose:  A boolean indicating whether to print verbose messages during
+            computation. Defaults to False.
+
+    """
+
+    def __init__(
+        self,
+        explanation_task: ExplanationTask,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize the Hyperparameter Interaction Game (HPIGame).
+
+        Args:
+            explanation_task: The `ExplanationTask` containing information about
+                the configuration space and surrogate model. This task defines the
+                hyperparameter search space and the model used to estimate performance.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        """
+        self.explanation_task = explanation_task
+        self.n_workers = n_workers if n_workers is not None else 1
+        self.verbose = verbose if verbose is not None else False
+
+        # determine the value of the empty coalition so that we can normalize wrt to that performance
+        normalization_value = self.evaluate_single_coalition(
+            np.array([False] * explanation_task.get_num_hyperparameters()),
+        )
+
+        super().__init__(
+            n_players=explanation_task.get_num_hyperparameters(),
+            normalize=True,
+            normalization_value=normalization_value,
+        )
+
+    def _process_chunk(self, chunk: np.ndarray) -> list:
+        """Process a chunk of coalitions, evaluating each coalition using the `evaluate_single_coalition` method.
+
+        Args:
+            chunk: A NumPy array representing a subset of coalitions.
+
+        Returns:
+            A list of floats, where each float is the value of a coalition
+            in the input chunk.
+
+        """
+        return [self.evaluate_single_coalition(c) for c in chunk]
+
+    def value_function(self, coalitions: np.ndarray) -> np.ndarray:
+        """Calculate the value of a list of coalitions.
+
+        This method handles both single-worker and multi-worker scenarios for efficient computation.
+
+        Args:
+            coalitions: A NumPy array representing a list of coalitions.  Each
+                coalition is a Boolean array indicating which hyperparameters
+                are included in that coalition.
+
+        Returns:
+            A NumPy array containing the values of the input coalitions.
+
+        """
+        if self.n_workers == 1:
+            value_list = []
+            for coalition in coalitions:
+                value_list += [self.evaluate_single_coalition(coalition)]
+        else:
+            m = len(coalitions)
+            num_workers = min(self.n_workers, m)
+            base_size = m // num_workers
+            remainder = m % num_workers
+
+            chunk_indices: Iterable[tuple[int, int]] = []
+            start = 0
+            for i in range(num_workers):
+                size = base_size + (1 if i < remainder else 0)
+                chunk_indices.append((start, start + size))
+                start += size
+
+            chunks = [coalitions[start:end] for start, end in chunk_indices]
+
+            with ThreadPoolExecutor(max_workers=self.n_workers) as executor:
+                partial_results = list(executor.map(self._process_chunk, chunks))
+
+            value_list = [val for sublist in partial_results for val in sublist]
+        return np.array(value_list)
+
+    @abstractmethod
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate the value of a single coalition.
+
+        This method *must* be implemented by subclasses.
+
+        Args:
+            coalition: A boolean array representing a coalition of hyperparameters.
+
+        Returns:
+            The value of the coalition (a float).
+
+        """
+
+    def get_num_hyperparameters(self) -> int:
+        """Return the number of hyperparameters being considered.
+
+        Returns:
+            The number of hyperparameters (an integer).
+
+        """
+        return self.explanation_task.get_num_hyperparameters()
+
+    def get_hyperparameter_names(self) -> list[str]:
+        """Return a list of the names of the hyperparameters.
+
+        Returns:
+            A list of strings, where each string is the name of a hyperparameter.
+
+        """
+        return self.explanation_task.get_hyperparameter_names()

hypershap/games/optimizerbias.py

@@ -0,0 +1,115 @@
+"""optimizerbias: Quantifying Bias in Hyperparameter Optimizers.
+
+This module provides tools for measuring and explaining bias in hyperparameter optimizers using cooperative game
+theory frameworks. It implements an explanation game specifically designed to assess how hyperparameter coalitions
+influence the performance of specific HPO algorithms compared to diverse alternatives.
+
+Key Features:
+- Provides a mechanism to measure optimizer biases through mathematical games
+- Enables identification of hyperparameters contributing most significantly to biased outcomes
+
+Classes:
+    OptimizerBiasGame: An explanation game class that quantifies bias in optimization algorithms by comparing
+    performance against an ensemble of diverse optimizers.
+
+Usage Notes:
+    This module requires integration with the HyperSHAP library and appropriate input data structures. It is designed
+    to help researchers understand and mitigate biases in machine learning optimization processes.
+"""
+
+from __future__ import annotations
+
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import numpy as np
+
+from hypershap.games import AbstractHPIGame
+from hypershap.task import OptimizerBiasExplanationTask
+
+
+class OptimizerBiasGame(AbstractHPIGame):
+    """An explanation game to measure bias in hyperparameter optimizers.
+
+    This class extends the `AbstractHPIGame` base class and is specifically designed
+    to quantify how much an optimizer is biased toward tuning certain hyperparameters.
+    To this end a set of diverse optimizers is used as a reference.
+
+    Attributes:
+        explanation_task (OptimizerBiasExplanationTask): The task that defines the game.
+            This includes the configuration sapce, the surrogate model and a baseline configuration, as well as the
+            optimizer of interest and the ensemble of diverse optimizers.
+
+
+    Methods:
+        evaluate_single_coalition: Computes the marginal contribution of a coalition (subset of features)
+            by comparing the optimizer of interest against an optimizer ensemble. This method is called internally
+            during the game's main evaluation process.
+
+    Note:
+        The game evaluates coalitions based on the difference between the outcome using the primary optimizer
+        and the maximum outcome achieved by any optimizer in the provided ensemble.
+
+    """
+
+    def __init__(
+        self,
+        explanation_task: OptimizerBiasExplanationTask,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize an instance of `OptimizerBiasGame`.
+
+        Args:
+            explanation_task (OptimizerBiasExplanationTask): The task that contains all necessary
+                information for defining the game. This includes the configuration space, the surrogate model, the
+                optimizer of interest, and the ensemble of diverse optimizers.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        Example:
+            >>> from hypershap.task import OptimizerBiasExplanationTask
+            >>> # Create an explanation task instance with specific parameters
+            >>> expl_task = OptimizerBiasExplanationTask(...)
+            >>> game = OptimizerBiasGame(expl_task)
+
+        """
+        super().__init__(explanation_task, n_workers=n_workers, verbose=verbose)
+
+    def _get_explanation_task(self) -> OptimizerBiasExplanationTask:
+        if isinstance(self.explanation_task, OptimizerBiasExplanationTask):
+            return self.explanation_task
+        raise ValueError
+
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate a single coalition by comparing against an optimizer ensemble.
+
+        Args:
+            coalition (np.ndarray): A binary array indicating which hyperparameters are included in the coalition.
+                The array has shape (n_hyperparameters,) where True means hyperparameters is included and False
+                 otherwise.
+
+        Returns:
+            float: The marginal contribution of the coalition. It is computed as the difference between
+                the outcome when using the optimizer of interest with the given coalition, versus the best outcome
+                achievable by any optimizer in the ensemble (including the one of interest).
+
+        Note:
+            This method overrides a base class abstract method and must be implemented to provide specific game logic.
+            The value returned here is used for computing Shapley values or other cooperative game theory measures.
+
+        Example:
+            >>> coalition = np.array([1, 0, 1])  # Hyperparameters 0 and 2 are included
+            >>> marginal_contribution = game.evaluate_single_coalition(coalition)
+            >>> print(marginal_contribution)  # Outputs the difference in outcomes for this coalition.
+
+        """
+        optimizer_res = self._get_explanation_task().optimizer_of_interest.search(coalition)
+        optimizer_ensemble_res = [
+            optimizer.search(coalition) for optimizer in self._get_explanation_task().optimizer_ensemble
+        ]
+        return optimizer_res - max([*optimizer_ensemble_res, optimizer_res])

hypershap/games/tunability.py

@@ -0,0 +1,183 @@
+"""Hyperparameter tunability games for analyzing the impact of tuning hyperparameters on performance.
+
+This module provides a suite of game-theoretic tools for analyzing the tunability
+of hyperparameters within a surrogate model of a black-box optimization
+process.  It defines classes that implement search-based games, allowing
+exploration of scenarios involving coalitions of hyperparameters and assessment
+of their impact on optimization performance.  The module leverages the
+`ExplanationTask` from `hypershap.task` to represent the hyperparameter
+search space and surrogate model, and provides flexible configuration options
+through the `ConfigSpaceSearcher` interface.
+
+The core functionality revolves around defining games like `TunabilityGame`,
+`SensitivityGame`, and `MistunabilityGame`, each representing a different
+aspect of hyperparameter behavior under coalition-based constraints. These
+games are built upon search strategies and allow for in-depth understanding of
+hyperparameter dependencies and potential for optimization gains.
+"""
+
+from __future__ import annotations
+
+import logging
+from typing import TYPE_CHECKING
+
+if TYPE_CHECKING:
+    import numpy as np
+
+    from hypershap.task import (
+        BaselineExplanationTask,
+        MistunabilityExplanationTask,
+        SensitivityExplanationTask,
+        TunabilityExplanationTask,
+    )
+
+from hypershap.games.abstract import AbstractHPIGame
+from hypershap.utils import ConfigSpaceSearcher, RandomConfigSpaceSearcher
+
+logger = logging.getLogger(__name__)
+
+
+class SearchBasedGame(AbstractHPIGame):
+    """Base class for games that rely on searching the configuration space."""
+
+    def __init__(
+        self,
+        explanation_task: BaselineExplanationTask,
+        cs_searcher: ConfigSpaceSearcher,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize the search-based game.
+
+        Args:
+            explanation_task: The explanation task containing the configuration
+                space and surrogate model.
+            cs_searcher: The configuration space searcher. If None, a
+                RandomConfigSpaceSearcher is used by default.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        """
+        self.cs_searcher = cs_searcher
+        super().__init__(explanation_task, n_workers=n_workers, verbose=verbose)
+
+    def evaluate_single_coalition(self, coalition: np.ndarray) -> float:
+        """Evaluate the value of a single coalition using the configuration space searcher.
+
+        Args:
+            coalition: A boolean array indicating which hyperparameters are
+                constrained by the coalition.
+
+        Returns:
+            The value of the coalition based on the search results.
+
+        """
+        return self.cs_searcher.search(coalition)
+
+
+class TunabilityGame(SearchBasedGame):
+    """Game representing the tunability of hyperparameters."""
+
+    def __init__(
+        self,
+        explanation_task: TunabilityExplanationTask,
+        cs_searcher: ConfigSpaceSearcher | None = None,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize the tunability game.
+
+        Args:
+            explanation_task: The explanation task containing the configuration
+                space and surrogate model.
+            cs_searcher: The configuration space searcher. If None, a
+                RandomConfigSpaceSearcher is used by default.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        """
+        # set cs searcher if not given by default to a random config space searcher.
+        if cs_searcher is None:
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="max")
+        elif cs_searcher.mode != "max":  # ensure that cs_searcher is maximizing
+            logger.warning("WARN: Tunability game set mode of given ConfigSpaceSearcher to maximize.")
+            cs_searcher.mode = "max"
+        super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)
+
+
+class SensitivityGame(SearchBasedGame):
+    """Game representing the sensitivity of hyperparameters."""
+
+    def __init__(
+        self,
+        explanation_task: SensitivityExplanationTask,
+        cs_searcher: ConfigSpaceSearcher | None = None,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize the sensitivity game.
+
+        Args:
+            explanation_task: The explanation task containing the configuration
+                space and surrogate model.
+            cs_searcher: The configuration space searcher. If None, a
+                RandomConfigSpaceSearcher is used by default.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        """
+        # set cs searcher if not given by default to a random config space searcher.
+        if cs_searcher is None:
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="var")
+        elif cs_searcher.mode != "var":  # ensure that cs_searcher is maximizing
+            logger.warning("WARN: Sensitivity game set mode of given ConfigSpaceSearcher to variance.")
+            cs_searcher.mode = "var"
+
+        super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)
+
+
+class MistunabilityGame(SearchBasedGame):
+    """Game representing the mistunability of hyperparameters."""
+
+    def __init__(
+        self,
+        explanation_task: MistunabilityExplanationTask,
+        cs_searcher: ConfigSpaceSearcher | None = None,
+        n_workers: int | None = None,
+        verbose: bool | None = None,
+    ) -> None:
+        """Initialize the mistunability game.
+
+        Args:
+            explanation_task: The explanation task containing the configuration
+                space and surrogate model.
+            cs_searcher: The configuration space searcher. If None, a
+                RandomConfigSpaceSearcher is used by default.
+            n_workers: The number of worker threads to use for parallel evaluation
+                of coalitions. Defaults to None meaning no parallelization.  Using more workers can significantly
+                speed up the computation of Shapley values.  The maximum number of workers is capped by the number of coalitions.
+            verbose:  A boolean indicating whether to print verbose messages during
+                computation. Defaults to None.  When set to True, the method prints
+                debugging information and progress updates.
+
+        """
+        # set cs searcher if not given by default to a random config space searcher.
+        if cs_searcher is None:
+            cs_searcher = RandomConfigSpaceSearcher(explanation_task, mode="min")
+        elif cs_searcher.mode != "min":  # ensure that cs_searcher is maximizing
+            logger.warning("WARN: Mistunability game set mode of given ConfigSpaceSearcher to minimize.")
+            cs_searcher.mode = "min"
+
+        super().__init__(explanation_task, cs_searcher, n_workers=n_workers, verbose=verbose)